Update board

README.md

@@ -15,14 +15,25 @@ tags:
 
 # MERA Reason Leaderboard
 
-Zip-only leaderboard for **ruAIME**, **T-math**, **Luzitania**, **MMReD**.
+Zip-only leaderboard for **ruAIME**, **T-math**, **Luzitania**, **MMReD** (open-reasoning track).
 
 ## Submission flow
 
-1. Upload a MERA-style zip on the **Submit** tab.
+1. Upload a MERA-style zip on the **Submit** tab (`ruaime.json`, `t_math.json`, `luzitania.json`, `mmred.json`, `logs_public/`).
 2. Metrics are extracted from `results_*.json` inside `logs_public`.
-3. Aggregated scores are saved to `results/{model}.json` and reloaded on every page visit.
-4. Tables update immediately — no HF datasets or evaluation queue.
+3. Aggregated scores are saved to the results dataset repo and reloaded on every page visit.
+4. Tables update immediately — no evaluation queue.
+
+## Metrics
+
+| Benchmark | Main leaderboard metric |
+|-----------|-------------------------|
+| ruAIME | Exact match |
+| T-math | Exact match |
+| Luzitania | Exact match |
+| MMReD | `em.dc_aggregate` |
+
+The MMReD dataset tab also shows group-level `em.dc_aggregate` and per-subtask `exact_match`.
 
 ## Configuration
 

deepseek-reason-8b.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e8eb5afc2528cc627ca4a49016ae381db1a74c98ee8dcfe53d1177bc93ceb176
-size 8450

gigachat-reason-xl.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b3791d8f89c0828ec4736704fdfebcc8989f519c006a85987556ba9044001d78
-size 8383

glm-5.1.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e724b19c59a3c4910019363f6af48726722dc69304e1ecdef47a9164fb9df25b
-size 21684272

glm-reason-9b.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a8f53568ded55448cf06447365b49d777898ed1d94c7b35464935bd60cfc3836
-size 8407

qwen-reason-7b.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:26f228894bdca7a255e75caac15bd620896596e52bd505ed3a58fb801f60d7e2
-size 8458

results/.gitkeep

@@ -1 +0,0 @@
-# Keep directory in git; submission JSON files are stored here.

results/GLM.json

@@ -1,26 +0,0 @@
-{
-  "model": "GLM",
-  "team": "MERA",
-  "submitted_time": "2026-06-18T13:14:20Z",
-  "ruaime": {
-    "dataset_total_score": 0.4165,
-    "pass@1": 0.433,
-    "em": 0.4
-  },
-  "t_math": {
-    "dataset_total_score": 0.4645,
-    "pass@1": 0.478,
-    "em": 0.451
-  },
-  "luzitania": {
-    "dataset_total_score": 0.598,
-    "pass@1": 0.612,
-    "em": 0.584
-  },
-  "mmred": {
-    "dataset_total_score": 0.52,
-    "pass@1": 0.533,
-    "em": 0.507
-  },
-  "total_score": 0.49975
-}

results/Qwen.json

@@ -1,26 +0,0 @@
-{
-  "model": "Qwen",
-  "team": "MERA",
-  "submitted_time": "2026-06-18T13:16:06Z",
-  "ruaime": {
-    "dataset_total_score": 0.35,
-    "pass@1": 0.367,
-    "em": 0.333
-  },
-  "t_math": {
-    "dataset_total_score": 0.39949999999999997,
-    "pass@1": 0.412,
-    "em": 0.387
-  },
-  "luzitania": {
-    "dataset_total_score": 0.511,
-    "pass@1": 0.524,
-    "em": 0.498
-  },
-  "mmred": {
-    "dataset_total_score": 0.4385,
-    "pass@1": 0.456,
-    "em": 0.421
-  },
-  "total_score": 0.42474999999999996
-}

src/about.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from enum import Enum
 
 
@@ -6,6 +6,7 @@ from enum import Enum
 class Metric:
     key: str
     col_name: str
+    description: str = ""
 
 
 @dataclass
@@ -16,6 +17,7 @@ class Task:
     description: str
     task_type: str
     test_size: int
+    dataset_url: str = ""
     primary_metric: str = ""
 
     def __post_init__(self):
@@ -28,69 +30,125 @@ class Task:
         return self.primary_metric
 
 
+MMRED_GROUP_DEFS = [
+    ("mmred_dc_sa_c", "DC-SA-C", "Кто провёл больше/меньше всего времени в одиночестве?"),
+    ("mmred_dc_sr_i", "DC-SR-I", "Сколько шагов X провёл в комнате Y?"),
+    ("mmred_dc_cc_i", "DC-CC-I", "Сколько шагов в комнатах существовала толпа (3+ человек)?"),
+    ("mmred_dc_ws_r", "DC-WS-R", "В какой комнате X провёл больше/меньше всего времени?"),
+    ("mmred_dc_whs_c", "DC-WHS-C", "Кто провёл больше/меньше всего времени в комнате X?"),
+]
+
+MMRED_LENGTHS = [32, 64, 128]
+
+MMRED_GROUP_KEYS = {group_key for group_key, _, _ in MMRED_GROUP_DEFS}
+
+MMRED_SUBTASK_KEYS = {
+    f"{group_key}_{length}"
+    for group_key, _, _ in MMRED_GROUP_DEFS
+    for length in MMRED_LENGTHS
+}
+
+
+def _mmred_metrics() -> list[Metric]:
+    metrics = [
+        Metric(
+            "em.dc_aggregate",
+            "Overall",
+            "Итоговая оценка MMReD: гармоническое среднее пяти взвешенных EM по типам вопросов.",
+        ),
+    ]
+    for group_key, group_label, group_desc in MMRED_GROUP_DEFS:
+        metrics.append(
+            Metric(
+                f"{group_key}::em.dc_aggregate",
+                group_label,
+                f"{group_desc} Взвешенный EM по длинам 32/64/128.",
+            )
+        )
+    for group_key, group_label, _ in MMRED_GROUP_DEFS:
+        for length in MMRED_LENGTHS:
+            metrics.append(
+                Metric(
+                    f"{group_key}_{length}::exact_match",
+                    f"{group_label} × {length}",
+                    f"Exact Match на подзадаче длины {length}.",
+                )
+            )
+    return metrics
+
+
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
     ruaime = Task(
         benchmark="ruaime",
         metrics=[
-            Metric("pass@1", "pass@1"),
-            Metric("em", "Exact match"),
+            Metric(
+                "exact_match",
+                "Exact match",
+                "Доля ответов, математически эквивалентных эталону (проверка через math_verify).",
+            ),
         ],
         col_name="ruAIME",
+        dataset_url="https://huggingface.co/datasets/MERA-evaluation/ruAIME",
         description=(
-            "Бенчмарк математического рассуждения на русском языке в стиле AIME. "
-            "Содержит олимпиадные задачи с числовыми ответами, требующие многошагового рассуждения. "
-            "Ответы проверяются автоматически с помощью библиотеки math_verify."
+            "Русскоязычный набор задач AIME (1983–2025): олимпиадная математика с числовыми "
+            "ответами, требующая многошагового рассуждения."
         ),
         task_type="Математика, рассуждение",
-        test_size=30,
+        test_size=724,
+        primary_metric="exact_match",
     )
     t_math = Task(
         benchmark="t_math",
         metrics=[
-            Metric("pass@1", "pass@1"),
-            Metric("em", "Exact match"),
+            Metric(
+                "exact_match",
+                "Exact match",
+                "Математическая эквивалентность предсказания и эталона (math_verify).",
+            ),
         ],
         col_name="T-math",
+        dataset_url="https://huggingface.co/datasets/MERA-evaluation/T-math",
         description=(
-            "Датасет олимпиадных математических задач на русском языке. "
-            "331 задача из Всероссийской олимпиады школьников и Московской олимпиады "
-            "(1998–2025). Задачи требуют многошагового рассуждения, ответы — числовые."
+            "331 олимпиадная математическая задача на русском из Всероссийской олимпиады "
+            "школьников и Московской олимпиады (1998–2025)."
         ),
         task_type="Математика, олимпиадные задачи",
         test_size=331,
+        primary_metric="exact_match",
     )
     luzitania = Task(
         benchmark="luzitania",
         metrics=[
-            Metric("pass@1", "pass@1"),
-            Metric("em", "Exact match"),
+            Metric(
+                "exact_match",
+                "Exact match",
+                "Доля ответов, точно совпадающих с эталонным числовым ответом.",
+            ),
         ],
         col_name="Luzitania",
+        dataset_url="https://huggingface.co/datasets/MERA-evaluation/Luzitania",
         description=(
-            "Бенчмарк логического и абстрактного рассуждения на русском языке. "
-            "Оценивает способность моделей решать задачи с множественным выбором "
-            "и свободным ответом в различных категориях рассуждений."
+            "Набор для оценки многошагового математического рассуждения на русском языке: "
+            "олимпиадные и продвинутые задачи с целочисленными ответами."
         ),
         task_type="Логика, рассуждение",
-        test_size=500,
+        test_size=252,
+        primary_metric="exact_match",
     )
     mmred = Task(
         benchmark="mmred",
-        metrics=[
-            Metric("pass@1", "pass@1"),
-            Metric("em", "Exact match"),
-        ],
+        metrics=_mmred_metrics(),
         col_name="MMReD",
+        dataset_url="https://huggingface.co/datasets/MERA-evaluation/MMReD",
         description=(
-            "Multi-Modal Reasoning in Dense Context — синтетический бенчмарк для оценки "
-            "рассуждения на длинном плотном контексте. Модели анализируют последовательности "
-            "состояний и отвечают на вопросы, требующие агрегации информации по всей последовательности. "
-            "5 типов вопросов × 3 длины контекста (32, 64, 128) = 750 вопросов."
+            "Multi-Modal Reasoning in Dense Context — синтетический бенчмарк рассуждения "
+            "на длинном плотном контексте. 5 типов вопросов × 3 длины (32, 64, 128) = 750 вопросов."
         ),
         task_type="Длинный контекст, рассуждение",
         test_size=750,
+        primary_metric="em.dc_aggregate",
     )
 
 
@@ -118,10 +176,10 @@ LLM_BENCHMARKS_TEXT = """
 
 | Бенчмарк | Тип | Размер | Основная метрика |
 |----------|-----|--------|------------------|
-| ruAIME | Математика | 30 | pass@1 |
-| T-math | Олимпиадная математика | 331 | pass@1 |
-| Luzitania | Логика, рассуждение | 500 | pass@1, Exact match |
-| MMReD | Длинный контекст | 750 | pass@1, Exact match |
+| ruAIME | Математика | 724 | Exact match |
+| T-math | Олимпиадная математика | 331 | Exact match |
+| Luzitania | Логика, рассуждение | 252 | Exact match |
+| MMReD | Длинный контекст | 750 | em.dc_aggregate |
 """
 
 SUBMIT_TEXT = """
@@ -132,13 +190,13 @@ SUBMIT_TEXT = """
 - JSON-файлы с предсказаниями для каждого бенчмарка (`ruaime.json`, `t_math.json`, `luzitania.json`, `mmred.json`)
 - `logs_public.zip` или папка `logs_public/` с файлами `results_*.json`
 
-Оценки берутся **только** из `results_*.json` внутри `logs_public`. После загрузки результат сохраняется в локальную папку `results/{model}.json` и подхватывается при перезагрузке страницы.
+Оценки берутся **только** из `results_*.json` внутри `logs_public`. После загрузки результат сохраняется в dataset-репозиторий и подхватывается при перезагрузке страницы.
 """
 
 CITATION_BUTTON_LABEL = "Скопируйте для цитирования"
 CITATION_BUTTON_TEXT = r"""
 @misc{mera_reason,
   title={MERA Reason: Russian Reasoning Evaluation Benchmark},
-  howpublished={\url{https://huggingface.co/spaces/MERA-Evaluation/leaderboard}},
+  howpublished={\url{https://huggingface.co/spaces/MERA-evaluation/MERA_Reason}},
 }
 """

src/leaderboard/metrics.py

@@ -1,18 +1,29 @@
+EXACT_MATCH_ALIASES = [
+    "exact_match",
+    "exact_match,flexible-extract",
+    "exact_match,strict-match",
+    "exact_match,scoring",
+]
+
+EM_DC_AGGREGATE_ALIASES = [
+    "em.dc_aggregate",
+    "em.dc_aggregate,scoring",
+]
+
 METRIC_ALIASES: dict[str, list[str]] = {
-    "pass@1": ["pass@1", "exact_match,extract_number", "exact_match,strict-match"],
-    "acc": ["acc", "exact_match,strict-match"],
-    "em": ["em", "exact_match,flexible-extract"],
-    "acc_32": ["acc_32"],
-    "acc_64": ["acc_64"],
-    "acc_128": ["acc_128"],
+    "exact_match": EXACT_MATCH_ALIASES,
+    "em.dc_aggregate": EM_DC_AGGREGATE_ALIASES,
+    "em": EXACT_MATCH_ALIASES,
 }
 
 
 def extract_metric_value(task_result: dict, metric_key: str) -> float | None:
-    if metric_key in task_result and isinstance(task_result[metric_key], (int, float)):
-        return float(task_result[metric_key])
+    base_key = metric_key.split("::")[-1]
+
+    if base_key in task_result and isinstance(task_result[base_key], (int, float)):
+        return float(task_result[base_key])
 
-    candidates = [metric_key] + METRIC_ALIASES.get(metric_key, [])
+    candidates = [base_key] + METRIC_ALIASES.get(base_key, [])
     for candidate in candidates:
         if candidate in task_result and isinstance(task_result[candidate], (int, float)):
             return float(task_result[candidate])
@@ -27,7 +38,6 @@ def extract_metric_value(task_result: dict, metric_key: str) -> float | None:
     for key, value in task_result.items():
         if not isinstance(value, (int, float)) or key in ("alias",) or ",stderr" in key:
             continue
-        base = key.split(",")[0]
-        if base == metric_key or key.startswith(f"{metric_key},"):
+        if key == base_key or key.startswith(f"{base_key},"):
             return float(value)
     return None

src/leaderboard/store.py

@@ -22,10 +22,12 @@ def has_submission(model: str) -> bool:
     return os.path.isfile(result_path_for_model(model))
 
 
-def _dataset_total_score(metrics: dict[str, float]) -> float:
-    if not metrics:
-        return 0.0
-    return sum(metrics.values()) / len(metrics)
+def _primary_metric_score(benchmark: str, metrics: dict[str, float]) -> float:
+    task = next(t for t in Tasks if t.value.benchmark == benchmark)
+    primary = task.value.primary_metric
+    if primary in metrics:
+        return float(metrics[primary])
+    return 0.0
 
 
 def build_result_payload(
@@ -47,7 +49,7 @@ def build_result_payload(
         if not raw_metrics:
             continue
         metrics = {key: float(value) for key, value in raw_metrics.items()}
-        dataset_total = _dataset_total_score(metrics)
+        dataset_total = _primary_metric_score(benchmark, metrics)
         dataset_scores.append(dataset_total)
         payload[benchmark] = {
             "dataset_total_score": dataset_total,

src/populate.py

@@ -61,8 +61,14 @@ def get_dataset_leaderboard_df(task_name: str) -> pd.DataFrame:
 
 def get_dataset_info_markdown(task_name: str) -> str:
     task = Tasks[task_name].value
+    dataset_link = (
+        f"[{task.col_name}]({task.dataset_url})"
+        if task.dataset_url
+        else task.col_name
+    )
     metrics_table = "\n".join(
-        f"| {m.col_name} | `{m.key}` |" for m in task.metrics
+        f"| {m.col_name} | `{m.key}` | {m.description or '—'} |"
+        for m in task.metrics
     )
     return f"""
 ### {task.col_name}
@@ -71,14 +77,16 @@ def get_dataset_info_markdown(task_name: str) -> str:
 
 | Параметр | Значение |
 |----------|----------|
+| Датасет | {dataset_link} |
 | Тип | {task.task_type} |
 | Размер теста | {task.test_size} |
 | Ключ в JSON | `{task.benchmark}` |
+| Основная метрика | `{task.primary_metric}` |
 
 #### Метрики
 
-| Отображение | Ключ в JSON |
-|-------------|-------------|
+| Отображение | Ключ в JSON | Описание |
+|-------------|-------------|----------|
 {metrics_table}
 """
 

src/submission/process_zip.py

@@ -5,7 +5,7 @@ import tempfile
 import zipfile
 from dataclasses import dataclass
 
-from src.about import Tasks
+from src.about import MMRED_GROUP_KEYS, MMRED_SUBTASK_KEYS, Tasks
 from src.leaderboard.metrics import extract_metric_value
 
 
@@ -13,22 +13,24 @@ LOGS_PUBLIC_ZIP = "logs_public.zip"
 LOGS_PUBLIC_DIR = "logs_public"
 RESULTS_PREFIX = "results_"
 
+
+@dataclass
+class ParsedResultFile:
+    path: str
+    benchmark: str
+    data: dict
+
+
 TASK_KEY_ALIASES: dict[str, set[str]] = {
     task.value.benchmark: {task.value.benchmark, task.value.col_name.lower()}
     for task in Tasks
 }
+TASK_KEY_ALIASES["t_math"].update({"tmath", "t-math"})
 for task in Tasks:
     benchmark = task.value.benchmark
     TASK_KEY_ALIASES[benchmark].add(benchmark.replace("_", ""))
 
 
-@dataclass
-class ParsedResultFile:
-    path: str
-    benchmark: str
-    data: dict
-
-
 def _normalize_task_key(task_key: str) -> str:
     key = task_key.lower().removesuffix("_gen")
     return key.replace("-", "_")
@@ -36,6 +38,10 @@ def _normalize_task_key(task_key: str) -> str:
 
 def _benchmark_for_task_key(task_key: str) -> str | None:
     normalized = _normalize_task_key(task_key)
+    if normalized == "tmath":
+        return "t_math"
+    if normalized == "mmred" or normalized.startswith("mmred_"):
+        return "mmred"
     for benchmark, aliases in TASK_KEY_ALIASES.items():
         alias_norm = {_normalize_task_key(alias) for alias in aliases}
         if normalized in alias_norm or normalized.startswith(benchmark):
@@ -52,25 +58,60 @@ def _submission_name_candidates(task) -> set[str]:
         benchmark.replace("_", "").lower(),
         col_name.replace("-", "").lower(),
         col_name.replace("-", "_").lower(),
+        "tmath",
     }
 
 
-def get_scores_from_result(parsed: ParsedResultFile) -> dict[str, float]:
-    task = next(t for t in Tasks if t.value.benchmark == parsed.benchmark)
-    task_result = None
-    for task_key, result in parsed.data.get("results", {}).items():
-        if _benchmark_for_task_key(task_key) == parsed.benchmark:
-            task_result = result
-            break
-    if task_result is None:
+def _extract_mmred_scores(task_key: str, task_result: dict, scores: dict[str, float]) -> None:
+    normalized = _normalize_task_key(task_key)
+    if normalized == "mmred":
+        value = extract_metric_value(task_result, "em.dc_aggregate")
+        if value is not None:
+            scores["em.dc_aggregate"] = value
+        return
+    if normalized in MMRED_GROUP_KEYS:
+        value = extract_metric_value(task_result, "em.dc_aggregate")
+        if value is not None:
+            scores[f"{normalized}::em.dc_aggregate"] = value
+        return
+    if normalized in MMRED_SUBTASK_KEYS:
+        value = extract_metric_value(task_result, "exact_match")
+        if value is not None:
+            scores[f"{normalized}::exact_match"] = value
+
+
+def _extract_benchmark_scores(task_key: str, task_result: dict) -> dict[str, float]:
+    benchmark = _benchmark_for_task_key(task_key)
+    if benchmark is None:
         return {}
 
-    scores = {}
-    for metric in task.value.metrics:
-        value = extract_metric_value(task_result, metric.key)
-        if value is not None:
-            scores[metric.key] = value
-    return scores
+    if benchmark == "mmred":
+        scores: dict[str, float] = {}
+        _extract_mmred_scores(task_key, task_result, scores)
+        return scores
+
+    value = extract_metric_value(task_result, "exact_match")
+    if value is None:
+        return {}
+    return {"exact_match": value}
+
+
+def parse_results_data(data: dict) -> dict[str, dict[str, float]]:
+    merged: dict[str, dict[str, float]] = {}
+    for task_key, task_result in data.get("results", {}).items():
+        if not isinstance(task_result, dict):
+            continue
+        benchmark = _benchmark_for_task_key(task_key)
+        if benchmark is None:
+            continue
+        scores = _extract_benchmark_scores(task_key, task_result)
+        if scores:
+            merged.setdefault(benchmark, {}).update(scores)
+    return merged
+
+
+def get_scores_from_result(parsed: ParsedResultFile) -> dict[str, float]:
+    return parse_results_data(parsed.data).get(parsed.benchmark, {})
 
 
 def extract_submission_archive(zip_path: str) -> str:
@@ -137,35 +178,51 @@ def validate_submission_files(submission_root: str) -> list[str]:
     return missing
 
 
-def parse_logs_public_results(logs_public_dir: str) -> list[ParsedResultFile]:
-    parsed = []
-    for name in sorted(os.listdir(logs_public_dir)):
-        if not name.startswith(RESULTS_PREFIX) or not name.endswith(".json"):
+def _iter_results_files(*roots: str):
+    seen = set()
+    for root in roots:
+        if not os.path.isdir(root):
             continue
-        path = os.path.join(logs_public_dir, name)
+        for name in sorted(os.listdir(root)):
+            if not name.startswith(RESULTS_PREFIX) or not name.endswith(".json"):
+                continue
+            path = os.path.join(root, name)
+            if path in seen:
+                continue
+            seen.add(path)
+            yield path
+
+
+def _count_results_files(*roots: str) -> int:
+    count = 0
+    for root in roots:
+        if not os.path.isdir(root):
+            continue
+        for name in os.listdir(root):
+            if name.startswith(RESULTS_PREFIX) and name.endswith(".json"):
+                count += 1
+    return count
+
+
+def parse_logs_public_results(logs_public_dir: str, submission_root: str | None = None) -> dict[str, dict[str, float]]:
+    merged: dict[str, dict[str, float]] = {}
+    roots = [logs_public_dir]
+    if submission_root:
+        roots.append(submission_root)
+
+    for path in _iter_results_files(*roots):
         with open(path, encoding="utf-8") as fp:
             data = json.load(fp)
-
-        for task_key, task_result in data.get("results", {}).items():
-            benchmark = _benchmark_for_task_key(task_key)
-            if benchmark is None:
-                continue
-            parsed.append(ParsedResultFile(path=path, benchmark=benchmark, data=data))
-            break
-    return parsed
+        for benchmark, scores in parse_results_data(data).items():
+            merged.setdefault(benchmark, {}).update(scores)
+    return merged
 
 
 def build_normalized_result_payload(
-    parsed_results: list[ParsedResultFile],
+    results: dict[str, dict[str, float]],
     model_name: str,
     team: str,
 ) -> dict:
-    results = {}
-    for parsed in parsed_results:
-        scores = get_scores_from_result(parsed)
-        if scores:
-            results[parsed.benchmark] = scores
-
     return {
         "model": model_name,
         "team": team,
@@ -177,24 +234,34 @@ def process_submission_zip(
     zip_path: str,
     model_name: str,
     team: str,
-) -> tuple[dict, list[ParsedResultFile], list[str]]:
+) -> tuple[dict, dict[str, dict[str, float]], list[str]]:
     extract_dir = extract_submission_archive(zip_path)
     logs_public_dir = None
     try:
         submission_root = _find_submission_root(extract_dir)
         missing_files = validate_submission_files(submission_root)
         logs_public_dir = _resolve_logs_public_dir(submission_root)
-        parsed_results = parse_logs_public_results(logs_public_dir)
+        parsed_results = parse_logs_public_results(logs_public_dir, submission_root)
 
         if not parsed_results:
+            roots = [logs_public_dir]
+            if submission_root:
+                roots.append(submission_root)
+            has_results = _count_results_files(*roots) > 0
+            if has_results:
+                detail = "Found results_*.json files but could not extract leaderboard metrics from them."
+            else:
+                detail = (
+                    "logs_public contains no results_*.json files (only samples are not enough). "
+                    "Re-pack the submission with log_to_reasoning_submission.py after evaluation."
+                )
             raise ValueError(
-                "No leaderboard task scores found in logs_public. "
+                f"No leaderboard task scores found in logs_public. {detail} "
                 f"Expected results for: {', '.join(t.value.col_name for t in Tasks)}."
             )
 
-        found_benchmarks = {parsed.benchmark for parsed in parsed_results}
         expected_benchmarks = {task.value.benchmark for task in Tasks}
-        missing_benchmarks = expected_benchmarks - found_benchmarks
+        missing_benchmarks = expected_benchmarks - set(parsed_results)
         if missing_benchmarks:
             missing_names = [
                 next(t.value.col_name for t in Tasks if t.value.benchmark == benchmark)
@@ -205,12 +272,13 @@ def process_submission_zip(
                 + ", ".join(missing_names)
             )
 
-        for parsed in parsed_results:
-            if not get_scores_from_result(parsed):
-                task_name = next(
-                    t.value.col_name for t in Tasks if t.value.benchmark == parsed.benchmark
+        for benchmark in expected_benchmarks:
+            scores = parsed_results.get(benchmark, {})
+            task = next(t for t in Tasks if t.value.benchmark == benchmark)
+            if task.value.primary_metric not in scores:
+                raise ValueError(
+                    f"Could not extract {task.value.primary_metric} for {task.value.col_name} from logs_public."
                 )
-                raise ValueError(f"Could not extract metrics for {task_name} from logs_public.")
 
         resolved_model = model_name.strip()
         if not resolved_model:

yandex-reason-pro.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:be3c9af2f36b8c533581e1a230a08dfe09af4eb77914fdbb2be4d590170ef5a8
-size 8410