"""Gradio component builders for the leaderboard.
Each function creates one piece of the UI inside the surrounding Blocks/render
context. No data-path logic lives here (see ``data_loading.py``) and no copy
lives here (see ``website_texts.py``).
"""
from __future__ import annotations
import datetime
import functools
import html
import json
import math
import os
import re
import textwrap
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, replace
from itertools import groupby
from pathlib import Path
from typing import TYPE_CHECKING
import gradio as gr
import pandas as pd
from markdown_it import MarkdownIt
import website_texts
from constants import (
Constants,
model_type_color,
model_type_emoji,
system_shade,
system_shade_color,
system_shade_label,
tag_emoji,
type_color,
type_symbols,
variant_color,
)
from data_loading import (
AXIS_NOTES,
BEYOND_FEW_DATASETS,
BEYOND_FEW_DATASETS_NOTE,
BEYOND_SUBSET_LABELS,
BEYOND_SUBSET_NOTE,
DATASET_DISTRIBUTION_LABELS,
DATASET_LABELS,
DATASET_SIZE_LABELS,
SYSTEM_CATEGORY_LABELS,
DATASET_SIZE_NOTE,
TASK_LABELS,
BeyondSubset,
LBContainer,
Subset,
commercial_flags,
leakage_datasets,
leakage_evidence,
leakage_methods,
NO_LEAKAGE_SUFFIX,
fits_at_predict_flags,
license_names,
load_leaderboard_csv,
parse_model,
static_url,
entrants_categories,
entrants_note,
unzip_png,
widest_entrants_key,
)
if TYPE_CHECKING:
from pages import LeaderboardPage
# --------------------------------------------------------------------------- #
# Full per-subset leaderboard table
# --------------------------------------------------------------------------- #
_IMPUTED_INFO = (
"We impute the performance for models that cannot run on all datasets due to"
" task or dataset size constraints. We impute with the performance of a"
" default RandomForest. We add a postfix [X% IMPUTED] to the model if any"
" results were imputed. The X% shows the percentage of datasets that were"
" imputed. In general, imputation negatively represents the model"
" performance, punishing the model for not being able to run on all datasets."
)
# Variants a model is evaluated in; all selected by default, and any combination is
# valid. Systems carry no variant tag of their own but tune and ensemble internally, so
# they belong with those.
VARIANT_VALUES = ["default", "tuned", "tuned + ensembled"]
# Columns that drive the filters or are folded into another cell; kept on the frame,
# never rendered as their own column.
_INTERNAL_COLUMNS = ["TypeName", "RefModel", "Imputed", "_variant", "_base", "_search"]
# Shown as "(+115/-110)" after the Elo value instead of as a column of its own.
_CI_COLUMN = "Elo 95% CI"
_ELO_COLUMN = "Elo [⬆️]"
# Rendered as a ✔️ badge on the model name, the way the overview does it.
_VERIFIED_COLUMN = "Verified"
# Always shown, so they are not offered in the column picker.
_ALWAYS_SHOWN = ["#", "Type", "Model"]
# Column header -> the metric-reference entry whose text becomes its hover hint, so
# the tooltips and the documented definitions cannot drift apart.
_LB_TOOLTIP_METRIC = {
_ELO_COLUMN: "🏆 Elo (ranking aggregation)",
"Score [⬆️]": "📊 Score",
"Improvability (%) [⬇️]": "📉 Improvability (%)",
"Rank [⬇️]": "🔢 Average Rank",
"Harmonic Rank [⬇️]": "🎯 Harmonic Rank",
"Median Train Time (s/1K) [⬇️]": "⏱️ Train / Predict Time (s/1K)",
"Median Predict Time (s/1K) [⬇️]": "⏱️ Train / Predict Time (s/1K)",
"Imputed (%) [⬇️]": "🧩 Imputed (%)",
}
_LB_FIXED_TOOLTIPS = {
"#": "Position in this subset's Elo ranking, as published.",
"Type": "Model family; see the legend above the table.",
"Model": (
"The model, its configuration variant in brackets, and ✔️ when the "
"implementation was verified by its authors or the maintainers. Links to the "
"implementation."
),
"Hardware": "The hardware the reported runtimes were measured on.",
}
# Which direction is good, per column, for the per-column heatmap. Columns absent
# from this map (#, Hardware) are left unshaded.
_LB_HIGHER_IS_BETTER = {
_ELO_COLUMN: True,
"Score [⬆️]": True,
"Rank [⬇️]": False,
"Harmonic Rank [⬇️]": False,
"Improvability (%) [⬇️]": False,
"Median Train Time (s/1K) [⬇️]": False,
"Median Predict Time (s/1K) [⬇️]": False,
"Imputed (%) [⬇️]": False,
}
# Runtimes span orders of magnitude, so shading them linearly paints every model
# the same green and only the slowest one red. Normalize those in log space.
_LB_LOG_SCALED = {
"Median Train Time (s/1K) [⬇️]",
"Median Predict Time (s/1K) [⬇️]",
}
# The marker the plot explorers put on an imputed method (see the generated
# *_explorer.html); reused here so the same thing looks the same everywhere.
IMPUTED_MARK = "‡"
# Per-column value formatting; everything else falls back to _format_value.
_LB_FORMATS = {
_ELO_COLUMN: lambda v: str(int(round(v))),
"Score [⬆️]": lambda v: f"{v:.3f}",
"Rank [⬇️]": lambda v: f"{v:.2f}",
"Harmonic Rank [⬇️]": lambda v: f"{v:.2f}",
"Improvability (%) [⬇️]": lambda v: f"{v:.2f}",
"Median Train Time (s/1K) [⬇️]": lambda v: f"{v:.2f}",
"Median Predict Time (s/1K) [⬇️]": lambda v: f"{v:.3f}",
"Imputed (%) [⬇️]": lambda v: f"{v:.1f}",
}
def _column_tooltip(column: str) -> str | None:
"""Hover hint for a leaderboard column header, or None when there is nothing to add."""
if column in _LB_FIXED_TOOLTIPS:
return _LB_FIXED_TOOLTIPS[column]
name = _LB_TOOLTIP_METRIC.get(column)
if not name:
return None
for metric in website_texts.METRICS:
if metric["name"] == name:
return f"{metric['details']} · Why we use it: {metric['why']}"
return None
def _format_value(column: str, value) -> str:
"""Display string for one cell; `data-sort` keeps the raw value for sorting."""
if value is None or (isinstance(value, float) and pd.isna(value)):
return "–"
formatter = _LB_FORMATS.get(column)
if formatter and isinstance(value, (int, float)) and not isinstance(value, bool):
return formatter(value)
if isinstance(value, float) and value.is_integer():
return str(int(value))
return str(value)
def filter_leaderboard(
df: pd.DataFrame,
*,
models: list[str],
variants: list[str],
show_imputed: bool,
search: str = "",
) -> pd.DataFrame:
"""Apply the table's row filters.
Pure so it can be tested without a Gradio context. `df` carries the internal
``_base`` / ``_variant`` / ``Imputed`` / ``_search`` columns added by
:func:`_prepare_leaderboard`. `models` is the selected individual models: family
chips are bulk selectors over that list, not a filter of their own. Column
selection is applied when rendering, not here.
"""
sub = df[df["_base"].isin(models)]
# Only narrow when the reader has actually deselected something, so a row whose
# variant we failed to classify can never be dropped silently.
if variants is not None and set(variants) != set(VARIANT_VALUES):
sub = sub[sub["_variant"].isin(variants)]
if not show_imputed:
sub = sub[~sub["Imputed"].astype(bool)]
term = (search or "").strip().lower()
if term:
sub = sub[sub["_search"].str.contains(re.escape(term), regex=True)]
return sub
def _prepare_leaderboard(raw: pd.DataFrame) -> pd.DataFrame:
"""Add the internal columns the filters and the renderer need."""
df = raw.copy()
parsed = [parse_model(str(m)) for m in raw["Model"]]
df["_base"] = [base for base, _, _ in parsed]
# A system has no variant tag of its own, but it tunes and ensembles internally, so the
# variant filter groups it with the tuned ensembles. Keyed off the family rather than the
# name, so this holds for every system and not only AutoGluon.
df["_variant"] = [
"tuned + ensembled" if not variant and type_name == Constants.system else variant
for (_, variant, _), type_name in zip(parsed, raw["TypeName"], strict=True)
]
df["_search"] = [
f"{base} {variant} {type_name}".lower()
for (base, variant, _), type_name in zip(parsed, raw["TypeName"], strict=True)
]
return df
def leaderboard_families(df: pd.DataFrame) -> list[tuple[str, list[str]]]:
"""(family, models) pairs present in `df`, in the legend's family order."""
families = []
for type_name in model_type_emoji:
models = sorted(set(df.loc[df["TypeName"] == type_name, "_base"]))
if models:
families.append((type_name, models))
return families
def fam_chip_colors(families: list[tuple[str, list[str]]]) -> str:
"""`--fam` per family, so each chiprow carries its family's colour."""
rules = [
f".ta-fam-{re.sub(r'[^a-z0-9]+', '-', type_name.lower())}"
f"{{--fam:{model_type_color.get(type_name, '#9e9e9e')};}}"
for type_name, _ in families
]
# The variant toggles take their colours from the Leaderboard Overview explorer.
# Generated from the same list the choices come from, so the nth-of-type index
# cannot drift from the option order.
rules += [
f".ta-variants label:nth-of-type({i + 1}){{--fam:{variant_color[value]};}}"
for i, value in enumerate(VARIANT_VALUES)
if value in variant_color
]
return "".join(rules)
def _group_handler(index: int, models: list[str], render):
"""A model chip group changed: bring its family chip in line, redraw the table."""
def handler(*values):
selected = values[index] or []
return gr.update(value=set(selected) == set(models)), render(*values)
return handler
def _family_handler(index: int, models: list[str], render):
"""A family chip was toggled: select or clear all of its models, redraw the table."""
def handler(checked, *values):
updated = list(values)
updated[index] = list(models) if checked else []
return gr.update(value=updated[index]), render(*updated)
return handler
def _heatmap_bounds(df: pd.DataFrame, columns: list[str]) -> dict[str, tuple[float, float]]:
"""Per-column (lo, hi) over the rendered rows, in the scale used for shading."""
bounds = {}
for column in columns:
if column not in _LB_HIGHER_IS_BETTER or column not in df.columns:
continue
values = pd.to_numeric(df[column], errors="coerce").dropna()
if column in _LB_LOG_SCALED:
values = values[values > 0]
values = values.apply(math.log10)
if len(values) < 2 or values.min() == values.max():
continue
bounds[column] = (float(values.min()), float(values.max()))
return bounds
def _heatmap_style(column: str, value: float, bounds: dict[str, tuple[float, float]]) -> str:
"""Inline background for one shaded cell, or "" when the column is not shaded."""
if column not in bounds:
return ""
lo, hi = bounds[column]
scaled = value
if column in _LB_LOG_SCALED:
if value <= 0:
return ""
scaled = math.log10(value)
frac_best = (scaled - lo) / (hi - lo)
if not _LB_HIGHER_IS_BETTER[column]:
frac_best = 1 - frac_best
frac_best = max(0.0, min(1.0, frac_best))
return f' style="{_heat_style(1 - frac_best)}"'
def _model_cell(row: pd.Series) -> str:
"""The Model cell, styled like the cross-subset overview's."""
color = type_color(row["TypeName"], system_shade(row["TypeName"], row.get("Tags")))
_, variant, url = parse_model(str(row["Model"]))
name = html.escape(row["_base"])
if variant:
name += f' ({html.escape(variant)})'
if str(row.get(_VERIFIED_COLUMN, "")).strip() == "✔️":
name += ' ✔️'
if bool(row.get("Imputed", False)):
name += (
f' {IMPUTED_MARK}'
)
if url:
return (
f'{name}'
f'↗'
)
return f'{name}'
def leaderboard_table_html(df: pd.DataFrame, columns: list[str], table_id: str) -> str:
"""Render the leaderboard as a sortable HTML table in the overview's style.
Every header is clickable (``taSortTable`` in ``main.py``'s head) and carries the
column's definition as a hover hint. Numeric cells put the raw value in
``data-sort`` and the formatted one in the text, which is what lets the Elo cell
show its confidence interval without that text taking part in the sort.
"""
metric_columns = [c for c in columns if c not in _ALWAYS_SHOWN]
header = [
'
#
',
'
Type
',
'
Model
',
]
for column in metric_columns:
tooltip = _column_tooltip(column)
label = html.escape(column)
marker = ""
if column == _ELO_COLUMN:
label += ' (95% CI)'
# Tells taExportTable to split the interval into its own CSV column.
marker = ' data-col="elo"'
classes = "ta-th-sort" + (" ta-th-info" if tooltip else "")
title = f' title="{html.escape(tooltip, quote=True)}"' if tooltip else ""
header.append(
f'
{label}
'
)
# No medals, unlike the cross-subset overview: this table is the full ranking and
# already carries the # column, so podium marks only added a competing ordering.
bounds = _heatmap_bounds(df, metric_columns)
body = []
for _, row in df.iterrows():
shade = system_shade(row["TypeName"], row.get("Tags"))
color = type_color(row["TypeName"], shade)
cells = [
f'
{row["#"]}
',
f'
'
f'{type_symbols(row["Type"], shade)}
',
# data-export="text": the CSV should carry the displayed name, not the
# lowercased key this cell sorts on.
f'
'
f"{_model_cell(row)}
",
]
for column in metric_columns:
value = row.get(column)
text = _format_value(column, value)
if text == "–":
cells.append('
–
')
continue
numeric = isinstance(value, (int, float)) and not isinstance(value, bool)
shade = _heatmap_style(column, float(value), bounds) if numeric else ""
if column == _ELO_COLUMN:
ci = row.get(_CI_COLUMN)
interval, ci_attr = "", ""
if ci is not None and not pd.isna(ci):
interval = f' ({html.escape(str(ci))})'
ci_attr = f' data-ci="{html.escape(str(ci), quote=True)}"'
cells.append(
f'
'
f"{text}{interval}
"
)
continue
sort_key = value if numeric else html.escape(str(value), quote=True)
cells.append(f'
{text}
')
body.append(f"
{''.join(cells)}
")
caption = f"{len(df)} row{'' if len(df) == 1 else 's'}"
shaded = "green is better, red is worse, per column" if bounds else "unshaded"
return (
f'
'
f"
{''.join(header)}
"
f"{''.join(body)}
"
f'
{caption} · click a column header to sort, hover one for its '
f"definition · {shaded} · {IMPUTED_MARK} marks a model with imputed results
"
)
def make_leaderboard(lb: LBContainer, *, opened: bool = False) -> None:
"""The full leaderboard table for one subset, as a card like the figure tiles, ``opened`` or
closed.
The table is a generated artifact (`leaderboard_table.html`, built by
`tabarena.plot.interactive.leaderboard_table`), embedded the same way as the
other interactive plots. It lives upstream so that it reuses the explorers'
family and variant colours, chip components and imputation markers rather than
reimplementing them here, where they drifted.
Its button shows and hides it in the browser (``main.taToggleTable``); its frame loads only
while it is shown. The download sits in the card's header but is the
frame's own (``main.taLeaderboardCsv``): only the frame knows the current filters, columns and
sort order. On TabArena the frame is also told which methods the latest releases added, for
its "new" badges (``data-new``, ``main.taSendNew``).
Subsets whose artifacts predate it fall back to :func:`make_leaderboard_gradio`,
this app's own table.
"""
url = lb.html_url("leaderboard_table")
if url is None:
make_leaderboard_gradio(lb)
return
uid = _panel_uid(lb, "leaderboard_table")
fresh = _recent_additions() if isinstance(lb.subset, Subset) else {}
attrs = f'data-new="{html.escape(json.dumps(fresh), quote=True)}" ' if fresh else ""
frame = _explorer_frame(
url, f"Full leaderboard table, {lb.name}", height=1000, extra_attrs=attrs, exclude=lb.excluded_methods()
)
caption = html.escape(website_texts.FIGURE_CAPTIONS["table"], quote=True)
gr.HTML(
f'
',
padding=False,
apply_default_css=False,
elem_classes="ta-flush",
)
def make_leaderboard_gradio(lb: LBContainer) -> None:
"""The fallback table, rendered by this app rather than embedded.
Kept for subsets whose artifacts were generated before `leaderboard_table.html`
existed. :func:`make_leaderboard` is the current path; prefer fixing the
upstream generator over this.
Replaces the third-party `gradio_leaderboard` widget (which pinned Gradio < 6)
with the same hand-rolled HTML table the cross-subset overview uses: type pills,
dotted-underline model links, tooltip-carrying headers, sticky header and scroll
box. Model selection mirrors the plot explorers' edit view — a family chip above
its models, in the family's colour. Sorting and CSV export are client-side
(`taSortTable` / `taExportTable`); the filters round-trip to rebuild the HTML.
"""
raw = lb.load_df()
has_imputed = bool(raw["Imputed"].any())
df = _prepare_leaderboard(raw)
if not has_imputed:
df = df.drop(columns=["Imputed (%) [⬇️]"])
# The CI and Verified columns are folded into the Elo and Model cells.
renderable = [
c
for c in df.columns
if c not in _INTERNAL_COLUMNS + [_CI_COLUMN, _VERIFIED_COLUMN]
]
optional = [c for c in renderable if c not in _ALWAYS_SHOWN]
families = leaderboard_families(df)
table_id = f"lb-{lb.subset.rel_path.replace('/', '-')}"
def render(*values) -> str:
"""Redraw the table from the controls' values, positional as Gradio passes them."""
count = len(families)
models = [m for group in values[:count] for m in (group or [])]
variants, show_imputed, columns, search = values[count : count + 4]
sub = filter_leaderboard(
df,
models=models,
variants=variants,
show_imputed=show_imputed or not has_imputed,
search=search,
)
chosen = [c for c in renderable if c in set(columns or []) or c in _ALWAYS_SHOWN]
return leaderboard_table_html(sub, chosen, table_id)
with gr.Column(elem_classes="ta-lb", elem_id=_panel_uid(lb, "leaderboard_table")):
gr.HTML(
# The table borrows the overview's stylesheet; inject it here rather than
# relying on the legend, which this card no longer draws (the family chips
# below say the same thing, in colour).
_OVERVIEW_CSS
+ f""
+ '
'
'Full leaderboard table'
'every column sorts · hover a header for what it '
"means"
# Same green as the figure panels' export controls (.ta-exportbtn).
f''
"
"
)
# Model selection, laid out like the Pareto explorer's chiprows: one row per
# family, the family chip toggling all of its models at once.
fam_toggles, model_groups = [], []
for type_name, models in families:
fam_class = f"ta-fam-{re.sub(r'[^a-z0-9]+', '-', type_name.lower())}"
with gr.Row(elem_classes="ta-chiprow"):
fam_toggles.append(
gr.Checkbox(
value=True,
label=f"{model_type_emoji.get(type_name, '')} {type_name} ×{len(models)}",
show_label=False,
container=False,
interactive=True,
elem_classes=["ta-famchip", fam_class],
scale=0,
min_width=210,
)
)
model_groups.append(
gr.CheckboxGroup(
choices=models,
value=models,
show_label=False,
container=False,
interactive=True,
elem_classes=["ta-chips", fam_class],
scale=1,
)
)
with gr.Row(elem_classes="ta-lb-controls"):
variant = gr.CheckboxGroup(
choices=VARIANT_VALUES,
value=VARIANT_VALUES,
label="⚙️ Variants",
interactive=True,
elem_classes=["ta-btns", "ta-variants"],
scale=3,
min_width=290,
)
show_imputed = gr.Checkbox(
value=True,
label=f"{IMPUTED_MARK} Include imputed",
info=_IMPUTED_INFO,
interactive=True,
visible=has_imputed,
elem_classes=["ta-btns", "ta-btns-imputed"],
scale=1,
min_width=190,
)
columns = gr.Dropdown(
choices=optional,
value=optional,
multiselect=True,
label="📋 Columns",
info="# / Type / Model always shown",
interactive=True,
scale=2,
min_width=220,
)
search = gr.Textbox(
label="🔍 Search",
placeholder="model or type…",
interactive=True,
scale=1,
min_width=150,
)
state = [*model_groups, variant, show_imputed, columns, search]
table = gr.HTML(render(*[c.value for c in state]))
# `.input` rather than `.change`: a family chip rewrites its group's value and
# each group rewrites its family chip, so reacting to programmatic changes too
# would let the two bounce off each other.
for index, (models, group, fam) in enumerate(
zip([m for _, m in families], model_groups, fam_toggles, strict=True)
):
group.input(
_group_handler(index, models, render),
state,
[fam, table],
api_visibility="private",
)
fam.input(
_family_handler(index, models, render),
[fam, *state],
[group, table],
api_visibility="private",
)
for control in (variant, show_imputed, columns, search):
control.input(render, state, table, api_visibility="private")
# --------------------------------------------------------------------------- #
# Per-subset figures
# --------------------------------------------------------------------------- #
def _explorer_frame(
url: str,
title: str,
height: int = 720,
extra_attrs: str = "",
exclude: list[str] | None = None,
) -> str:
"""A sandboxed frame for one interactive explorer page, loaded from the page's static URL.
The page is a file Gradio serves (``gr.set_static_paths`` in ``main``), so a render sends this
tag rather than the page itself, and the browser keeps the page cached between renders. The URL
goes in ``data-src``: ``main.taLoadFrames`` sets ``src`` once the frame nears the screen, with
the site's theme in the URL's hash, so a frame further down the page or in a closed accordion
costs nothing until it is read.
The sandbox runs the page's scripts and lets it download its exports, nothing else: without
``allow-same-origin`` the frame stays cross-origin, so the host talks to it over postMessage.
The explorer posts its content height and the listener in ``main.py``'s head sizes the frame
to fit (``height`` is the placeholder until then); paper view, exports, the metric, the time
axis and the exclusions travel the other way (``main.taPaperView``, ``main.taExport``,
``main.taSendMetric`` and its siblings).
``extra_attrs`` is spliced into the tag for frames the host configures once they are up (a
Pareto tile's time axis, the per-dataset browser's opening filters). ``exclude`` names the
methods the frame should drop (see ``main.taSendExclude``), as a ``data-exclude`` JSON attribute.
"""
if exclude:
extra_attrs += f' data-exclude="{html.escape(json.dumps(sorted(exclude)), quote=True)}"'
return (
f''
)
def _panel_uid(lb: LBContainer, key: str) -> str:
"""A DOM-safe id unique to one (subset, figure) panel."""
return "ta-fig-" + re.sub(r"[^a-z0-9]+", "-", f"{lb.subset.rel_path}-{key}".lower()).strip("-")
def _download_menu(uid: str, label: str = "Download") -> str:
"""A panel's download button, opening the formats its figure exports to (main.taExport)."""
formats = "".join(
f""
for fmt in ("svg", "pdf", "png")
)
return (
f'{html.escape(label)}'
f'
{formats}
'
)
# What the reader is optimizing for -> which figure answers it first, and (for the Pareto
# panels) which time axis they plot against. Every Pareto explorer is generated from the same
# points upstream, so switching axis is a different view, not a different computation.
# Insertion order = selector order; first = default.
# Prediction speed before training speed: it is what serving costs, so it is the constraint
# more readers arrive with. End-to-end time is both together.
CARE_LABELS = {
"quality": "🏆 Best quality",
"infer": "⚡ Fast predictions",
"train": "⏱️ Fast to train",
"total": "🏁 Fast end-to-end",
}
# The second half of "I care about": which headline metric leads. Elo and Improvability
# answer different questions, and the leaderboard has always reported both.
METRIC_LABELS = {
"elo": "🏅 Consistent wins",
"imp": "📉 Relative gains",
}
METRIC_NOTES = {
"elo": (
"Ranks by Elo: how reliably a method beats the others head to head, whatever the margin."
),
"imp": (
"Ranks by Improvability: how far a method sits from the best one on each dataset, in percent."
),
}
CARE_NOTES = {
"quality": "Ranked purely on accuracy. Start here if compute is not your constraint.",
"infer": "Leads with the accuracy-versus-prediction-time trade-off, which is what serving costs.",
"train": "Leads with the accuracy-versus-training-time trade-off.",
"total": "Leads with the accuracy-versus-end-to-end-time trade-off: training and prediction together.",
}
def has_end_to_end(lb: LBContainer) -> bool:
"""Whether the subset's Pareto explorer offers end-to-end (train + inference) time.
Explorers generated before that axis existed do not (BeyondArena's, for now), and no
end-to-end panel is shown for them.
"""
return _offers_x_total(str(lb.base_path / "pareto_front_explorer.html"))
@functools.cache
def _offers_x_total(path: str) -> bool:
try:
return '"key": "x_total"' in Path(path).read_text(encoding="utf-8")
except FileNotFoundError:
return False
@dataclass(frozen=True)
class _Tile:
"""One figure of the grid under "Your leaderboard".
``role`` decides which headline metric the tile shows: ``"overview"`` and ``"scatter"`` tiles
take one each (see ``main.taTileMetric``), ``"matrix"`` has none. ``html_name`` is the explorer
page and ``img_name`` the static figure a subset may ship instead; ``xaxis`` is the time axis
a Pareto explorer opens on.
"""
key: str
role: str
title: str
html_name: str
img_name: str
xaxis: str = ""
wide: bool = False
#: Every figure the grid can hold, in its order when the reader cares about quality; "I care about"
#: moves another one to the front in the browser (the `.ta-page[data-care]` rules in main.py), so
#: no choice there costs a render. Inference and end-to-end time are one explorer page, which
#: offers every time axis; each tile's frame is told which one to open on (`main.taSendXAxis`).
_TILES = (
_Tile(
"overview",
"overview",
"Leaderboard Overview",
"leaderboard_overview_explorer",
"tuning-impact-elo",
wide=True,
),
_Tile(
"infer",
"scatter",
"Pareto Front, inference time",
"pareto_front_explorer",
"pareto_front_improvability_vs_time_infer",
xaxis="x_infer",
),
_Tile("total", "scatter", "Pareto Front, end-to-end time", "pareto_front_explorer", "", xaxis="x_total"),
_Tile(
"train",
"scatter",
"Pareto Front, train time",
"pareto_front_explorer",
"pareto_front_improvability_vs_time_train",
xaxis="x_train",
),
_Tile("trajectories", "scatter", "Tuning Trajectories", "tuning_trajectories_explorer", "pareto_n_configs_imp"),
_Tile("winrate", "matrix", "Win-rate Matrix", "winrate_explorer", "winrate_matrix", wide=True),
)
#: The figures of each view that shows them, by their ``_TILES`` keys.
VIEW_TILES = {
"leaderboard": ("overview",),
"time": ("infer", "total", "train", "trajectories"),
"h2h": ("winrate",),
}
def _figure_tiles(lb: LBContainer) -> list[_Tile]:
"""The tiles this subset has a figure for.
A tile shows when its explorer or its static figure ships, and the end-to-end one only where
the Pareto explorer offers that axis. The scatter tiles pair up two to a row; an odd one out
spans the row, so the grid has no hole whatever "I care about" puts first.
"""
tiles = []
for tile in _TILES:
if tile.key == "total" and not has_end_to_end(lb):
continue
if lb.html_url(tile.html_name) is None and not (tile.img_name and lb.has_image(tile.img_name)):
continue
tiles.append(tile)
scatter = [tile for tile in tiles if tile.role == "scatter"]
if len(scatter) % 2:
tiles[tiles.index(scatter[-1])] = replace(scatter[-1], wide=True)
return tiles
def _tile_html(lb: LBContainer, tile: _Tile) -> str:
"""One tile: a header row (title, caption, actions) above the figure.
The figure is the explorer where the subset ships one, with a switch to the static figure
when that ships too (``main.taSwitchView``); a subset whose artifacts predate the explorers
shows the static figure alone. The caption shows only where it fits beside the title whole
(``main.taFitCaptions``); the title's hover text carries it either way.
"""
uid = _panel_uid(lb, tile.key)
url = lb.html_url(tile.html_name)
image = lb.image_path(tile.img_name) if tile.img_name and lb.has_image(tile.img_name) else None
actions = ""
if url and image:
actions += (
f'"
)
if url:
# The tile opens in paper view, the figure alone; this opens its controls, and
# `aria-pressed` tracks whether paper view is on. The downloads show in paper view, the
# figure an export is made from.
actions += (
f'" + _download_menu(uid)
)
expand = html.escape(website_texts.TILE_EXPAND, quote=True)
actions += (
f'"
)
caption = html.escape(website_texts.FIGURE_CAPTIONS[tile.key], quote=True)
views = ""
alt = html.escape(f"{tile.title}, {lb.name}", quote=True)
if url:
frame = _explorer_frame(
url,
f"{tile.title} [{lb.name}]",
height=320 if tile.key == "overview" else 800 if tile.key == "winrate" else 500,
extra_attrs=f'data-xaxis="{tile.xaxis}" ' if tile.xaxis else "",
exclude=lb.excluded_methods(),
)
views += f'
{frame}
'
if image:
hidden = " ta-hidden" if url else ""
views += (
f'
{views}'
)
def make_figure_grid(lb: LBContainer, keys: Sequence[str]) -> None:
"""The subset's figures of one view, by their ``_TILES`` keys, as one grid: the overview and
the win rates span the width, the scatter plots (the three Pareto fronts and the tuning
trajectories) sit two to a row.
One HTML block, so "I care about" and "Measured by" reach it without a render: the page
carries the reader's choice as ``data-care`` / ``data-metric`` (``main.taSelector`` writes
them), the stylesheet orders the tiles from those, and ``main.taSendMetric`` tells each
explorer which metric to open on.
"""
tiles = [tile for tile in _figure_tiles(lb) if tile.key in keys]
no_total = "" if any(tile.key == "total" for tile in tiles) else " ta-no-total"
gr.HTML(
f'
' + "".join(_tile_html(lb, tile) for tile in tiles) + "
",
padding=False,
apply_default_css=False,
elem_classes="ta-flush",
)
#: The dataset browser's icon: a list of datasets.
_PD_ICON = (
''
)
def make_per_dataset_block(
lb: LBContainer,
*,
source_subset: Subset | BeyondSubset | None = None,
filter_attrs: str | None = None,
) -> None:
"""The per-dataset browser, the page's Datasets view: a panel like the figure tiles, open.
The artifact is published once per (entrants, imputation, splits) rather than for all 60
task/dataset cells: a dataset's own numbers do not depend on which *other* datasets share
its leaderboard, so the browser is read from the unrestricted cell and told which slice the
reader currently has selected. It filters itself from there, and the reader can widen it
again from inside without leaving the page. Its frame loads only once the view is shown
(main.taLoadFrames), since the page it embeds carries every dataset's results and is the
largest artifact the site ships.
``source_subset`` and ``filter_attrs`` are for BeyondArena, whose cells have no
``tasks="all", datasets="all"`` sibling: it names the cell that carries the browser and the
``data-`` attributes that tell the frame which slice is selected.
"""
uid = _panel_uid(lb, "per_dataset_explorer")
source = LBContainer(
data_root=lb.data_root,
subset=source_subset or replace(lb.subset, tasks="all", datasets="all"),
name=lb.name,
)
if filter_attrs is None:
# The dataset axis carries either a size or a distribution; the browser has a filter for
# each, so the value goes to the one it belongs to.
filter_attrs = (
f'data-task="{html.escape(lb.subset.tasks, quote=True)}" '
f'data-size="{lb.subset.datasets if lb.subset.datasets in DATASET_SIZE_LABELS else "all"}" '
f'data-balance="{lb.subset.datasets if lb.subset.datasets in DATASET_DISTRIBUTION_LABELS else "all"}" '
)
url = source.html_url("per_dataset_explorer")
teaser = html.escape(website_texts.PER_DATASET_TEASER, quote=True)
n = source.n_datasets
caption = website_texts.PER_DATASET_CAPTION.format(n=n) if n else website_texts.FIGURE_CAPTIONS["per_dataset"]
with gr.Column(elem_classes="ta-figpanel", elem_id=uid):
gr.HTML(
f'
'
f"{website_texts.PER_DATASET_TITLE}"
f'{html.escape(caption)}'
f'{_download_menu(uid, "Download the chart") if url else ""}
'
)
if url is None:
gr.HTML(f'
{website_texts.PER_DATASET_MISSING}
')
return
# `-i` matches what `taExport` and `taPaperView` look for on the other panels.
with gr.Column(elem_id=f"{uid}-i", elem_classes=["ta-figview", "ta-figview-flush"]):
gr.HTML(
_explorer_frame(
url,
f"Per-dataset results, {lb.name}",
height=1020,
# Read back by `main.taSendPerDatasetFilter` once the frame reports its
# height, so the browser opens on the slice selected above it.
extra_attrs=filter_attrs,
exclude=lb.excluded_methods(),
)
)
# --------------------------------------------------------------------------- #
# Cross-subset overview (Elo heatmap, rendered as HTML for links + grouping)
# --------------------------------------------------------------------------- #
# Columns of the overview: (label, group, subset, value_column). Always imputation=yes /
# splits=all. `value_column` is None for the subset columns, which report whichever metric the
# selector picked; the two cost columns pin their own instead, so the table reads as
# "how good, and what it costs" before it fans out per subset.
_TRAIN_TIME_COLUMN = "Median Train Time (s/1K) [⬇️]"
_PREDICT_TIME_COLUMN = "Median Predict Time (s/1K) [⬇️]"
#: Hover text of the † on a fit time that includes the prediction call (`FitsAtPredict`).
_FIT_AT_PREDICT_HINT = (
"This system fits inside its prediction call, so its fit time is the time of that call, "
"from training data to predictions. Its predict time holds the fit too."
)
#: The dagger's key under the train-time header.
_FIT_AT_PREDICT_KEY = "fit inside predict"
_FIXED_COLUMN_SPECS = {
"Fit (s/1K)": (_TRAIN_TIME_COLUMN, False, lambda v: f"{v:.2f}"),
"Infer (s/1K)": (_PREDICT_TIME_COLUMN, False, lambda v: f"{v:.3f}"),
}
_OVERVIEW_COLUMNS: list[tuple[str, str, Subset, str | None]] = [
("Overall", "overall", Subset(tasks="all", datasets="all"), None),
("Fit (s/1K)", "cost", Subset(tasks="all", datasets="all"), _TRAIN_TIME_COLUMN),
("Infer (s/1K)", "cost", Subset(tasks="all", datasets="all"), _PREDICT_TIME_COLUMN),
("Class.", "task", Subset(tasks="classification", datasets="all"), None),
("Regr.", "task", Subset(tasks="regression", datasets="all"), None),
("Binary", "task", Subset(tasks="binary", datasets="all"), None),
("Multi.", "task", Subset(tasks="multiclass", datasets="all"), None),
("Small", "size", Subset(tasks="all", datasets="small"), None),
("Medium", "size", Subset(tasks="all", datasets="medium"), None),
("Balanced", "target", Subset(tasks="all", datasets="balanced"), None),
("Imbalanced", "target", Subset(tasks="all", datasets="imbalanced"), None),
("Extreme", "target", Subset(tasks="all", datasets="extreme"), None),
]
_GROUP_TITLE = {"overall": "Overall", "cost": "What it costs", "task": "By Task", "size": "By Dataset Size", "target": "By Target"}
#: The overview's column groups that start folded into a stub column (main.taOverviewTable opens them).
_FOLDING_GROUPS = ("task", "size", "target")
#: How the Method header's key names a system's tag (constants.tag_emoji), which follows its name.
_SHADE_KEY = {"closed-source-api": "API system", "with-llm": "with LLMs"}
# Hover tooltips for the overview's subset column headers (rendered as a native `title=`).
# Size definitions are reused from DATASET_SIZE_NOTE so they can't drift from the subset blurbs.
_COLUMN_TOOLTIPS: dict[str, str] = {
"Overall": "All tasks across every dataset size. This is the headline ranking.",
"Fit (s/1K)": (
"Median seconds to train per 1000 rows, across all datasets, for the variant shown. "
"Tuning fits 200 configurations, so a tuned variant costs far more than its default. "
"Lower is better."
),
"Infer (s/1K)": (
"Median seconds to predict per 1000 rows, across all datasets, for the variant shown. "
"Lower is better."
),
"Class.": "Classification tasks only (binary + multiclass).",
"Regr.": "Regression tasks only.",
"Binary": "Binary classification tasks only.",
"Multi.": "Multiclass classification tasks only.",
"Small": DATASET_SIZE_NOTE["small"],
"Medium": DATASET_SIZE_NOTE["medium"],
"Balanced": DATASET_SIZE_NOTE["balanced"],
"Imbalanced": DATASET_SIZE_NOTE["imbalanced"],
"Extreme": DATASET_SIZE_NOTE["extreme"],
}
#: The top three of each overview column get a medal, in a slot every number cell reserves so the
#: digits line up whether or not one is present (`.ta-medal` in main.py). The full leaderboard table
#: has no medals: its # column already says the same thing.
_MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"}
#: The organization logos, served as static files (main.py registers the folder).
LOGO_DIR = Path(__file__).parent / "assets" / "logos"
#: Organization (the `Organization` column, from the upstream `MethodMetadata.organization`) ->
#: its logo in `LOGO_DIR`. An organization without one shows its initial instead.
ORGANIZATION_LOGOS = {
"Amazon": "amazon.png",
"fast.ai": "fastai.png",
"Google Research": "google.png",
"Inria": "soda-inria.png",
"Layer 6 AI": "layer6.png",
"Lexsi Labs": "lexsi-labs.png",
"LG AI Research": "lg.svg",
"Microsoft": "microsoft.png",
"Nanjing University": "lamda.png",
"Nums AI": "nums-ai.png",
"NVIDIA": "nvidia.png",
"Perpetual ML": "perpetual-ml.png",
"Prior Labs": "priorlabs.png",
"SAP": "sap.png",
"scikit-learn": "scikit-learn.png",
"Stable AI & Tsinghua University": "limix.png",
"Stanford University": "stanford.png",
"Synthefy": "synthefy.png",
"Technion": "technion.svg",
"UC San Diego": "ucsd.png",
"University of Freiburg": "freiburg.png",
"University of Stuttgart & Inria": "inria.png",
"University of Technology Nuremberg": "utn.png",
"University of Washington": "uw.svg",
"Xiaomi": "xiaomi.png",
"Yandex": "yandex.png",
"Yandex Research": "yandex.png",
"Yhat Labs": "yhat-labs.svg",
}
#: Methods that show their own project's logo rather than their organization's, by name prefix.
METHOD_LOGOS = {"AutoGluon": "autogluon.png"}
def org_logo(organization: str, method: str = "") -> str:
"""A method's organization as a small tile: its logo, its initial without one, blank when unknown.
A method named in `METHOD_LOGOS` shows that logo instead, with the organization in its hover.
"""
project = next((p for p in METHOD_LOGOS if method.startswith(p)), None)
if project is not None:
name = html.escape(f"{project} ({organization})" if organization else project, quote=True)
logo = METHOD_LOGOS[project]
elif not organization:
return ''
else:
name = html.escape(organization, quote=True)
logo = ORGANIZATION_LOGOS.get(organization)
if logo is None:
return f'{name[:1].upper()}'
# Relative to the working directory, which is how Gradio resolves a static file's path.
src = (Path(os.path.relpath(LOGO_DIR)) / logo).as_posix()
return (
f''
f''
)
# The Summary's two metrics, the ones the figures show too: key -> (csv column, higher_is_better,
# formatter). The headline group shows both; a breakdown one at a time, Elo first.
_OVERVIEW_METRIC_SPECS: dict[str, tuple] = {
"elo": ("Elo [⬆️]", True, lambda v: str(int(round(v)))),
"imp": ("Improvability (%) [⬇️]", False, lambda v: f"{v:.1f}"),
}
# What each metric means, in its headline column's hover text.
OVERVIEW_METRIC_TLDR = {
"elo": "Pairwise win-rate rating (a 400-point gap ≈ a 91% win rate). Higher is better.",
"imp": "How much lower the best model's error is than this one's, per dataset. Lower is better.",
}
#: Selector value -> the overview column it points at. A selection of "all" points at nothing,
#: because the Overall column already is that view.
_TASK_TO_COLUMN = {"classification": "Class.", "regression": "Regr.", "binary": "Binary", "multiclass": "Multi."}
_DATASET_TO_COLUMN = {
"small": "Small",
"medium": "Medium",
"balanced": "Balanced",
"imbalanced": "Imbalanced",
"extreme": "Extreme",
}
#: "I care about" -> the cost columns it points at; end-to-end time is the fit and the prediction together.
_CARE_TO_COLUMNS = {
"train": ("Fit (s/1K)",),
"infer": ("Infer (s/1K)",),
"total": ("Fit (s/1K)", "Infer (s/1K)"),
}
def _selected_overview_columns(tasks: str, datasets: str, care: str) -> list[str]:
"""The overview columns matching the current selectors, in the order they should lead."""
wanted = [*_CARE_TO_COLUMNS.get(care, ()), _TASK_TO_COLUMN.get(tasks), _DATASET_TO_COLUMN.get(datasets)]
return [c for c in wanted if c]
_SUBSET_FIELDS = [
"base", "TypeName", "shade", "variant", "url", "verified", "imputed", "commercial", "license",
"fits_at_predict", "organization", "ci", "leakage", "leakage_datasets",
]
def _subset_rows(df: pd.DataFrame, column: str) -> pd.DataFrame:
"""A subset's rows, `Model` split into base/variant/url and `column` renamed to `val`.
Systems stay in. Which entrants compete is decided by the pool the reader selected, and
the overview reports the numbers computed for that pool, so dropping a competitor here
would show a field the numbers were not computed against.
"""
df = df.dropna(subset=[column]).copy()
parsed = df["Model"].map(parse_model)
df["base"] = [p[0] for p in parsed]
df["variant"] = [p[1] for p in parsed]
df["url"] = [p[2] for p in parsed]
df["imputed"] = df["Imputed"].astype(bool) if "Imputed" in df.columns else False
df["verified"] = df["Verified"] if "Verified" in df.columns else ""
df["commercial"] = commercial_flags(df)
df["license"] = license_names(df)
df["fits_at_predict"] = fits_at_predict_flags(df)
# Artifacts generated before the column existed have no organizations.
df["organization"] = df["Organization"].fillna("").astype(str) if "Organization" in df.columns else ""
df["ci"] = df[_CI_COLUMN].fillna("").astype(str) if _CI_COLUMN in df.columns else ""
df["leakage"] = leakage_evidence(df)
df["leakage_datasets"] = leakage_datasets(df)
tags = df["Tags"] if "Tags" in df.columns else pd.Series(None, index=df.index)
df["shade"] = [system_shade(t, v) for t, v in zip(df["TypeName"], tags)]
return df.rename(columns={column: "val"})[[*_SUBSET_FIELDS, "val"]]
def _overview_th(
text: str, *, tip: str = "", selected: bool = False, attrs: str = "", key: str = "", classes: Sequence[str] = ()
) -> str:
"""A column header `
` named ``text``, with ``tip`` as its hover text and a 'help' affordance.
``attrs`` are further attributes, such as the overview's sort data, and ``classes`` further
classes. ``key`` (HTML) says under the name what the marks in the column mean.
"""
attrs = f" {attrs}" if attrs else ""
classes = [*classes, *(["ta-th-info"] if tip else []), *(["ta-col-sel"] if selected else [])]
if classes:
attrs += f' class="{" ".join(classes)}"'
if tip and "data-col" in attrs:
tip += " Click to sort by it."
if tip:
attrs += f' title="{html.escape(tip, quote=True)}"'
# The name has a span of its own, so the sort arrow (main.py) follows it rather than the key.
return f'{text}{_th_key(key)}
'
def _th_key(key: str) -> str:
"""What the marks in a column mean, as a muted line under its header's name (HTML).
The break is explicit because the headers do not wrap.
"""
return f' {key}' if key else ""
#: Shown after the name of a method whose license does not permit commercial use.
NONCOMMERCIAL_MARK = "$"
def noncommercial_mark(license_name: str = "") -> str:
"""The marker and its hover text, naming the license when one was declared."""
hint = "License does not permit commercial use"
if license_name:
hint += f": {license_name}"
return f' {NONCOMMERCIAL_MARK}'
#: The leakage warning's sign: a red triangle with an exclamation mark, drawn the same in both themes.
LEAKAGE_ICON = (
''
)
#: In the medal slot of a flagged method's quality numbers.
_LEAKAGE_NUMBER = (
f'!'
)
def leakage_badge(name: str, warning: tuple[str, Sequence[str]]) -> str:
"""The red badge after a flagged method's name, for ``warning`` as (evidence link, proven
datasets); its hover card (main.taLeakageCard) gives the evidence, the datasets and a way to
hide the flagged methods."""
url, datasets = warning
proven = website_texts.LEAKAGE_PROVEN[min(len(datasets), 2)].format(n=len(datasets))
text = website_texts.LEAKAGE_METHOD_TEXT.format(proven=proven)
attrs = {
"data-lk": "method",
"data-lk-head": f"{website_texts.LEAKAGE_METHOD_HEAD}: {name}",
"data-lk-text": text,
"data-lk-more": website_texts.LEAKAGE_DATASETS.format(", ".join(datasets)) if datasets else "",
"data-lk-url": url,
"data-lk-link": website_texts.LEAKAGE_EVIDENCE,
"data-lk-hide": website_texts.LEAKAGE_HIDE_ONE,
}
data = " ".join(f'{key}="{html.escape(value, quote=True)}"' for key, value in attrs.items())
label = html.escape(attrs["data-lk-head"], quote=True)
return (
f'"
)
def _leakage_key(rows_meta: Iterable[dict]) -> str:
"""The Method header's key for the leakage warning; its hover card speaks for the whole table."""
n = sum(1 for m in rows_meta if m["leakage"])
head = website_texts.LEAKAGE_PAGE_HEAD[1 if n == 1 else 2].format(n=n)
attrs = {
"data-lk": "page",
"data-lk-head": head,
"data-lk-text": website_texts.LEAKAGE_PAGE_TEXT if n == 1 else website_texts.LEAKAGE_PAGE_TEXT_MANY,
"data-lk-hide": website_texts.LEAKAGE_HIDE_ONE if n == 1 else website_texts.LEAKAGE_HIDE,
}
data = " ".join(f'{key}="{html.escape(value, quote=True)}"' for key, value in attrs.items())
return (
f'"
)
def _shade_scale(value: float, *, log_scale: bool) -> float | None:
"""The value in the space the heatmap shades in, or ``None`` when it cannot be placed.
The cost columns are shaded in log space, as the full leaderboard table already does
(`_LB_LOG_SCALED`): runtimes span four orders of magnitude, and a linear ramp painted every
model the same green with only the slowest one red, so a 10x speed difference near the fast
end was invisible. A non-positive runtime has no logarithm and gets the neutral shade.
"""
if not log_scale:
return float(value)
if value <= 0:
return None
return math.log10(value)
_HEAT_STOPS = [(0.0, (28, 120, 62)), (0.5, (138, 122, 36)), (1.0, (160, 58, 58))]
# The same ramp for the opt-in light theme (main.LIGHT_CSS): each stop keeps its hue and is lifted
# to a pastel that dark digits read on. Cells carry it as `--ta-heat-light`, which only the light
# theme reads; the leaderboard table frame carries the same stops upstream.
HEAT_STOPS_LIGHT = [(0.0, (135, 201, 160)), (0.5, (224, 204, 123)), (1.0, (223, 144, 144))]
# The Summary's in-cell bars, whose colour says how good a value is: amber for the best (0) through
# violet to a muted violet for the worst (1), per theme. Hue and lightness change together, so the
# ramp reads without telling red from green, and it stands apart from the heat cells after the
# divider. The dark theme's amber is deep enough to keep the white digits on it at 4.3:1.
BAR_STOPS = [(0.0, (158, 104, 10)), (0.5, (138, 90, 168)), (1.0, (74, 60, 110))]
BAR_STOPS_LIGHT = [(0.0, (242, 194, 90)), (0.5, (199, 163, 221)), (1.0, (221, 211, 236))]
def _interp_color(frac: float, stops: list = _HEAT_STOPS) -> str:
"""Map 0 (best) .. 1 (worst) to a green->olive->red hex (by default, readable on dark bg)."""
frac = max(0.0, min(1.0, frac))
for (f0, c0), (f1, c1) in zip(stops, stops[1:]):
if frac <= f1:
t = 0 if f1 == f0 else (frac - f0) / (f1 - f0)
r, g, b = (round(a + (b_ - a) * t) for a, b_ in zip(c0, c1))
return f"#{r:02x}{g:02x}{b:02x}"
return "#{:02x}{:02x}{:02x}".format(*stops[-1][1])
def _heat_style(frac: float) -> str:
"""The inline style of a shaded cell: the dark shade, plus the light one for the light theme."""
return f"background:{_interp_color(frac)};color:#f7f7f7;--ta-heat-light:{_interp_color(frac, HEAT_STOPS_LIGHT)};"
_OVERVIEW_CSS = """
"""
# --------------------------------------------------------------------------- #
# The selector card
# --------------------------------------------------------------------------- #
@dataclass(frozen=True)
class Choice:
"""One chip of a selector row.
Neighbouring chips of one ``section`` share a group, under the caption that key has in
``SELECTOR_CAPTIONS``.
``locked`` says why the chip cannot be clicked: one that is always on, or a category nobody has
submitted to yet. ``on`` is a toggle's or switch's state on load.
"""
value: str
label: str
hint: str = ""
section: str = ""
locked: str = ""
on: bool = False
@dataclass(frozen=True)
class ChoiceRow:
"""One row of the selector card: a caption and the chips that set the selection field ``key``.
``kind`` is "one" (pick one chip; the first is the default), "many" (toggle any number) or
"switch" (each chip is an on/off setting of its own, named by its value). A ``counted`` row
shows on each chip how many datasets the leaderboard would cover with that chip picked.
"""
key: str
caption: str
hint: str
choices: Sequence[Choice]
kind: str = "one"
counted: bool = False
def _plain(label: str) -> str:
"""A chip label as the spec line names it: no leading emoji, and lower-case unless an acronym."""
text = re.sub(r"^[^\w(]+", "", label)
return text[:1].lower() + text[1:] if text[1:2].islower() else text
def _selector_chip(row: ChoiceRow, choice: Choice, picked: bool) -> str:
if row.kind == "one":
state = f'role="radio" aria-checked="{str(picked).lower()}"'
elif row.kind == "many":
state = f'aria-pressed="{str(choice.on).lower()}"'
else:
state = f'role="switch" aria-checked="{str(choice.on).lower()}"'
if choice.locked:
state += ' aria-disabled="true" data-locked'
mark = "" if row.kind == "one" else ''
count = '' if row.counted else ""
return (
f''
)
def _selector_row(row: ChoiceRow, inline: bool = False) -> str:
# Runs of neighbouring chips of one section; a run with a section is a group.
runs: list[tuple[str, list[str]]] = []
for i, choice in enumerate(row.choices):
if not runs or choice.section != runs[-1][0]:
runs.append((choice.section, []))
runs[-1][1].append(_selector_chip(row, choice, picked=i == 0))
chips = []
for section, run in runs:
if section:
caption = (
f''
f"{html.escape(website_texts.SELECTOR_CAPTIONS[section])}"
)
run = [f'{caption}{"".join(run)}']
chips += run
group = ' role="radiogroup"' if row.kind == "one" else ' role="group"'
return (
f'
"
f'{html.escape(row.caption)}'
f'
{"".join(chips)}
'
)
#: The selector bar's icon: three sliders, drawn like the per-dataset bar's list (_PD_ICON).
_SELECTOR_ICON = (
''
)
def _selector_examples(rows: dict[str, ChoiceRow], examples: Sequence[tuple[str, str, str, object]]) -> str:
"""The card bar's shortcuts, as (label, hover text, row, value), each a button for taSelector.
A shortcut names a row of the card and the chip it picks, or in a row of toggles a tuple of the
chips it turns on together, or a switch by its value and the state it puts it in (True or False).
Locked chips, such as a system category nobody has submitted to, are left out, and so is a
shortcut with none left; one that names no chip of the card is a mistake in the texts.
"""
switches = {choice.value: choice for row in rows.values() if row.kind == "switch" for choice in row.choices}
buttons = []
for label, hint, field, value in examples:
if field in rows and rows[field].kind != "switch":
wanted = value if isinstance(value, tuple) and rows[field].kind == "many" else (value,)
chips = {c.value: c for c in rows[field].choices}
choices = [chips.get(v) for v in wanted]
state = " ".join(c.value for c in choices if c and not c.locked)
elif field in switches and isinstance(value, bool):
choices, state = [switches[field]], str(value).lower() if not switches[field].locked else ""
else:
choices, state = [None], ""
if None in choices:
raise ValueError(f"The selector shortcut {label!r} names no chip of the card: {field}={value!r}.")
if not state:
continue
buttons.append(
f''
)
return "".join(buttons)
def selector_card(
page: str,
lines: Sequence[Sequence[ChoiceRow]],
*,
title: str,
lead: str,
counts: dict[str, int],
count_axes: Sequence[str],
anchor: str,
warnings: Sequence[tuple[str, str, str, str]] = (),
examples: Sequence[tuple[str, str, str, object]] = (),
) -> gr.HTML:
"""Every selector of a page, as one card of chips that answers a click at once.
The card starts closed, as one row like the per-dataset browser's bar: its ``title``, which is the
button that opens it, the ``examples`` (see :func:`_selector_examples`), shortcuts that pick a
chip and open the card on it, and at its end how many datasets the selection covers, open or
closed. Open, the ``lead`` heads the ``lines`` under the bar. Each line holds one row, or several
that read as one choice, the later ones with their caption inline. A click updates the card in the browser
(main.taSelector): the chip, the dataset counts and the ``warnings``, as (selection
field, value, headline, details in Markdown), each shown while that value is selected. The whole
selection then goes to Python as one ``pick`` event, whose listener writes the page's states.
``counts`` maps the values of ``count_axes`` joined by "|" to the number of datasets that
leaderboard covers; a chip whose combination is missing cannot be picked.
"""
body = "".join(
f'
{"".join(_selector_row(row, inline=i > 0) for i, row in enumerate(line))}
'
for line in lines
)
rows = {row.key: row for line in lines for row in line}
shortcuts = _selector_examples(rows, examples)
if shortcuts:
shortcuts = (
f''
f"{shortcuts}"
)
# The whole bar opens and closes the card (main.taSelector); the title is the button a keyboard
# reaches, its chevron the one that shows it opens.
bar = (
'
'
f'{_SELECTOR_ICON}'
''
f"{shortcuts}
"
)
warns = "".join(
f''
f'⚠️{html.escape(head)}'
''
f'
{render_markdown(body)}
'
for field, value, head, body in warnings
)
data = {
"counts": counts,
"axes": list(count_axes),
"none": website_texts.SELECTOR_NO_DATASETS,
"total": website_texts.SELECTOR_TOTAL,
# Every dataset of the page: each row on its first chip.
"whole": counts.get("|".join(rows[axis].choices[0].value for axis in count_axes)),
}
return gr.HTML(
f'
'
f'{bar}
{html.escape(lead)}
{body}
'
'
'
'
'
f'
{warns}
',
js_on_load="taSelector(element, (selection) => trigger('pick', selection));",
apply_default_css=False,
elem_id=anchor,
elem_classes="ta-sel-block",
)
#: The view bar's anchor per page.
VIEW_ANCHORS = {"tabarena": "ta-views-anchor", "beyondarena": "beyond-views-anchor"}
def view_bar(slug: str, *, follow: str = "", hint: str = "", extra: str = "") -> str:
"""A page's view tabs (``website_texts.VIEW_TABS``), the top of its controls, with ``follow`` (the
Follow picker, HTML) at the right.
Plain buttons: the browser shows the view a tab names (``main.taShowView``), every view being on
the page from the start, so a switch costs no round trip, and marks the page's shown one
whenever the bar is drawn again (``main.taSyncViewTabs``). A tab's hover text says what its view
shows, ending with ``hint``, what the selection covers; ``extra`` (HTML) follows the tabs, such as
a warning, and so does a note that only one view has, shown with it.
"""
hints = dict(website_texts.VIEW_HINTS)
notes = {}
if slug == "beyondarena":
hints.update(website_texts.BEYOND_VIEW_HINTS)
# Its Summary charts the subsets, whose meaning is a Learn more topic.
notes["summary"] = (
f'"
)
buttons = "".join(
f'"
for i, (key, label) in enumerate(website_texts.VIEW_TABS)
)
shown = "".join(f'{note}' for key, note in notes.items())
return (
f'
'
f''
f'
{shown}{extra}
{follow}
'
)
def picked(value: object, labels: dict[str, str]) -> str:
"""``value`` when it is one of the ``labels`` keys, else the first key (the row's default).
The selection comes from the browser, so nothing in it is used before it is checked.
"""
return value if isinstance(value, str) and value in labels else next(iter(labels))
#: Anchor on TabArena's selector card.
CONTROLS_ANCHOR = "ta-controls-anchor"
def make_cross_subset_overview(
data_root: Path,
entrants: str = "models",
tasks: str = "all",
datasets: str = "all",
care: str = "quality",
include_noncommercial: bool = True,
) -> gr.HTML:
"""The Summary: every entrant (rows) on every subset (columns), within one pool, as one table.
A row is one model *variant* — default, tuned, or tuned + ensembled — so its quality and its
cost are always the same run's.
The headline columns, Elo and improvability over all datasets, and the cost columns carry an
in-cell bar as long as the value is high in its column, coloured from amber for the best
values to violet for the worst (BAR_STOPS); a divider follows each of the two groups. The breakdowns by task, size and
target are heat cells, folded into stub columns until opened, and carry both
metrics, of which a switch in each one's heading shows one (main.taOverviewTable).
`entrants` selects the pool. Every column is read from that pool's artifacts, so the whole
grid is one consistent field of competitors.
`tasks`, `datasets` and `care` do not change what the table contains, only where the eye
lands: the column each of them points at leads its group and is tinted, and its breakdown
starts open, so the reader can find their own view without hunting along the header.
Reordering happens inside a group rather than across the whole table, which would leave the
grouped header describing columns that had moved out from under it.
`include_noncommercial` off drops the rows whose license forbids commercial use. A row
filter, unlike the entrant pool: the remaining rows keep their published numbers, which
were computed against the full field. A method with a leakage warning (its `Leakage` column)
carries a badge whose hover gives the evidence, a mark in the medal slot of each quality number,
and no medal: each medal goes to the best method without a warning. Leaving those methods out
is a pool of its own (`data_loading.leakage_pool`), recomputed without them.
"""
# A column is (id, label, group, metric): a cost column reports its own number and has no
# metric; every other one comes twice, once per metric, its id the label and the metric.
columns: list[tuple[str, str, str, str | None]] = []
vals: dict[str, dict[tuple[str, str], float]] = {}
imps: dict[str, dict[tuple[str, str], bool]] = {}
cis: dict[str, dict[tuple[str, str], str]] = {}
meta: dict[tuple[str, str], dict] = {}
for label, group, subset, value_column in _OVERVIEW_COLUMNS:
path = Path(data_root) / replace(subset, entrants=entrants).rel_path / "website_leaderboard.csv"
if not path.exists():
continue
df = load_leaderboard_csv(str(path.resolve()))
specs = (
[(label, None, value_column)]
if value_column is not None
else [(f"{label}|{m}", m, spec[0]) for m, spec in _OVERVIEW_METRIC_SPECS.items()]
)
for cid, metric, csv_column in specs:
if csv_column not in df.columns:
continue
rows = _subset_rows(df, csv_column)
if not include_noncommercial:
rows = rows[rows["commercial"]]
if rows.empty:
continue
keys = list(zip(rows["base"], rows["variant"]))
vals[cid] = dict(zip(keys, rows["val"]))
imps[cid] = dict(zip(keys, rows["imputed"]))
# The Elo columns carry their bootstrap interval.
if metric == "elo":
cis[cid] = dict(zip(keys, rows["ci"]))
columns.append((cid, label, group, metric))
for key, (_, row) in zip(keys, rows.iterrows()):
meta.setdefault(
key,
{
"type_name": row["TypeName"],
"shade": row["shade"],
"variant": row["variant"],
"url": row["url"],
"verified": row["verified"],
"commercial": bool(row["commercial"]),
"license": row["license"],
"fits_at_predict": bool(row["fits_at_predict"]),
"organization": row["organization"],
"leakage": (row["leakage"], row["leakage_datasets"]) if row["leakage"] else None,
},
)
if not columns:
return _no_overview()
# Each selected column leads its own group; a breakdown runs its Elo columns, then the same
# columns in improvability.
highlighted = [c for c in _selected_overview_columns(tasks, datasets, care) if any(c == col[1] for col in columns)]
folding = [g for g in _FOLDING_GROUPS if any(col[2] == g for col in columns)]
ordered = []
for group in dict.fromkeys(col[2] for col in columns):
items = [col for col in columns if col[2] == group]
items = [col for col in items if col[1] in highlighted] + [col for col in items if col[1] not in highlighted]
if group in folding:
items = [col for m in _OVERVIEW_METRIC_SPECS for col in items if col[3] == m]
ordered += items
columns = ordered
def higher(col: tuple) -> bool:
return _FIXED_COLUMN_SPECS[col[1]][1] if col[3] is None else _OVERVIEW_METRIC_SPECS[col[3]][1]
def fmt(col: tuple):
return _FIXED_COLUMN_SPECS[col[1]][2] if col[3] is None else _OVERVIEW_METRIC_SPECS[col[3]][2]
lead = next((col for col in columns if col[2] == "overall" and col[3] == "elo"), columns[0])
worst = float("-inf") if higher(lead) else float("inf")
entries = sorted(meta, key=lambda k: vals[lead[0]].get(k, worst), reverse=higher(lead))
rank_by_col, medal_by_col, bounds = {}, {}, {}
for col in columns:
values = vals[col[0]]
ranked = sorted(values, key=lambda b: values[b], reverse=higher(col))
rank_by_col[col[0]] = {b: i + 1 for i, b in enumerate(ranked)}
# A method with a leakage warning wins no medal; the next one without a warning does.
clean = [b for b in ranked if not meta[b]["leakage"]]
medal_by_col[col[0]] = {b: _MEDALS[i + 1] for i, b in enumerate(clean[:3])}
shaded = [_shade_scale(v, log_scale=col[3] is None) for v in values.values()]
shaded = [v for v in shaded if v is not None]
bounds[col[0]] = (min(shaded), max(shaded)) if shaded else (0.0, 1.0)
# A divider closes each group before the breakdowns: the headline metrics, then the costs.
before = [col for col in columns if col[2] not in folding]
ends = {[c for c in before if c[2] == g][-1][0]: g for g in dict.fromkeys(c[2] for c in before)}
auto_open = " ".join(dict.fromkeys(col[2] for col in columns if col[1] in highlighted and col[2] in folding))
def data_attrs(col: tuple) -> str:
return f' data-g="{col[2]}" data-m="{col[3]}"' if col[2] in folding else ""
# -- Grouped header (two rows); the model column stays in view when the table scrolls sideways,
# the rank column sliding under it. A row's family shows in the colour of its name, keyed in
# the Method header, so there is no type column.
# Every number column sorts on a click (main.taOverviewTable), best first; the headline Elo
# is the order the rows arrive in.
rows_meta = meta.values()
marks = " ".join(
f'{mark}'
for mark, shown in (
('✔️ verified', any(m["verified"] == "✔️" for m in rows_meta)),
(f'{NONCOMMERCIAL_MARK} non-commercial', any(not m["commercial"] for m in rows_meta)),
('↗ paper or code', any(m["url"] for m in rows_meta)),
(_leakage_key(rows_meta), any(m["leakage"] for m in rows_meta)),
*(
(f'{tag_emoji[shade]} {html.escape(_SHADE_KEY[shade])}',
any(m["shade"] == shade for m in rows_meta))
for shade in _SHADE_KEY
),
)
if shown
)
column_keys = {}
if any(any(col.values()) for col in imps.values()):
column_keys[lead[0]] = '* imputed'
if any(m["fits_at_predict"] for m in rows_meta):
column_keys["Fit (s/1K)"] = f'† {_FIT_AT_PREDICT_KEY}'
row1 = [
f'
",
]
row2 = []
for group, items in groupby(columns, key=lambda col: col[2]):
items = list(items)
sep = " ta-xo-sep" if group in ends.values() else ""
if group in folding:
switch = "".join(
f'"
for m in _OVERVIEW_METRIC_SPECS
)
row1.append(
f'
'
f'{_GROUP_TITLE[group]}{switch}'
f'
'
)
else:
row1.append(f'
{_GROUP_TITLE[group]}
')
for i, col in enumerate(items):
cid, label, _, metric = col
if metric is None:
text, tip = label, _COLUMN_TOOLTIPS.get(label, "")
elif group == "overall":
text = website_texts.OVERVIEW_METRIC_NAMES[metric][0]
tip = f"{OVERVIEW_METRIC_TLDR[metric]} {_COLUMN_TOOLTIPS['Overall']}"
else:
text = label
tip = f"{_COLUMN_TOOLTIPS.get(label, '')} In {website_texts.OVERVIEW_METRIC_NAMES[metric][0].lower()}."
attrs = (
f'data-col="{columns.index(col)}" data-label="{html.escape(cid, quote=True)}" '
f'data-dir="{"desc" if higher(col) else "asc"}"'
+ (f' aria-sort="{"descending" if higher(col) else "ascending"}"' if col == lead else "")
+ data_attrs(col)
)
row2.append(
_overview_th(
text,
tip=tip,
selected=label in highlighted,
attrs=attrs,
key=column_keys.get(cid, ""),
classes=("ta-xo-sep",) if cid in ends else (),
)
)
for group in folding:
n = sum(1 for col in columns if col[2] == group) // len(_OVERVIEW_METRIC_SPECS)
row1.append(
f'
' for group in folding)
added = _recent_additions()
# -- Body
body = []
for key in entries:
m = meta[key]
color = type_color(m["type_name"], m["shade"])
# The name and what follows it each stay whole; a long one moves its suffix to a second line.
suffix = f'({html.escape(m["variant"])})' if m["variant"] else ""
if m["shade"] in tag_emoji:
suffix += (
f' '
f"{tag_emoji[m['shade']]}"
)
if m["verified"] == "✔️":
suffix += ' ✔️'
if not m["commercial"]:
suffix += noncommercial_mark(m["license"])
if m["url"]:
suffix += '↗'
name = f'{html.escape(key[0])}{suffix}'
if m["url"]:
name = (
f'{name}'
)
else:
name = f'{name}'
if key[0] in added:
name += f'new'
if m["leakage"]:
name += leakage_badge(key[0], m["leakage"])
cells = [
f'
{rank_by_col[lead[0]].get(key, "–")}
',
(
f'
{org_logo(m["organization"], key[0])}{name}'
# Follows the method in the figures below (main.taToggleFollow); outside the link.
f'
"
),
]
for col in columns:
cid, label, group, metric = col
classes = ["ta-num"]
if cid in ends:
classes.append("ta-xo-sep")
if label in highlighted:
classes.append("ta-col-sel")
val = vals[cid].get(key)
if val is None:
cells.append(f'
–
')
continue
# Leakage inflates quality, not cost: only the quality numbers carry its mark.
if m["leakage"] and metric is not None:
medal = _LEAKAGE_NUMBER
else:
medal = medal_by_col[cid].get(key, "")
lo, hi = bounds[cid]
scaled = _shade_scale(val, log_scale=metric is None)
if hi <= lo or scaled is None:
frac_best = 0.5
else:
frac_best = (scaled - lo) / (hi - lo) if higher(col) else (hi - scaled) / (hi - lo)
rank = rank_by_col[cid][key]
hover = f"#{rank} of {len(rank_by_col[cid])} · {_COLUMN_NAMES.get(label, label)}"
if metric is not None:
hover += f" · {website_texts.OVERVIEW_METRIC_NAMES[metric][0].lower()}"
ci = cis.get(cid, {}).get(key, "")
if ci:
hover += f" · 95% CI {ci}"
imp = '*' if imps[cid].get(key) else ""
if label == "Fit (s/1K)" and m["fits_at_predict"]:
imp += f'†'
if group in folding:
style = _heat_style(1 - frac_best)
else:
# Before the dividers a bar as long as the value sits high in its column (on the log
# scale for the costs), so a lower improvability or a faster time is a shorter bar;
# its colour (BAR_STOPS) says whether that is good. The headline columns read larger, and
# the headline Elo shows its interval.
classes.append("ta-bar")
classes.append("ta-xo-lead" if group == "overall" else "ta-xo-cost")
length = 0.5 if hi <= lo or scaled is None else (scaled - lo) / (hi - lo)
style = (
f"--bar-dark:{_interp_color(1 - frac_best, BAR_STOPS)};"
f"--bar-light:{_interp_color(1 - frac_best, BAR_STOPS_LIGHT)};"
f"--w:{max(4, round(length * 100))}%;"
)
ci = f'{html.escape(ci.replace("-", "−"))}' if ci and col == lead else ""
cells.append(
f'
'
f'{medal}'
f'{fmt(col)(val)}{imp}{ci}
'
)
flagged = ' class="ta-lk-row"' if m["leakage"] else ""
body.append(
f'
{"".join(cells)}{stubs}
'
)
table = f'
{header}{"".join(body)}
'
return gr.HTML(
f'{_OVERVIEW_CSS}
{table}
',
js_on_load="taOverviewTable(element);",
apply_default_css=False,
elem_classes="ta-overview-block",
)
def _no_overview() -> gr.HTML:
"""A note in place of the overview, when its pool has no data."""
return gr.HTML("
No overview data available.
", elem_classes="ta-overview-block")
#: An overview column as its hover text names it.
_COLUMN_NAMES = {
"Overall": "all datasets",
"Fit (s/1K)": "train time",
"Infer (s/1K)": "inference time",
"Class.": "classification",
"Regr.": "regression",
"Binary": "binary",
"Multi.": "multiclass",
"Small": "small datasets",
"Medium": "medium datasets",
"Balanced": "balanced targets",
"Imbalanced": "imbalanced targets",
"Extreme": "extreme imbalance",
}
#: How many of the latest releases that added a method the "new" badge covers.
_NEW_RELEASES = 3
def _family_key(type_name: str, shade: str | None) -> str:
"""The legend entry a row belongs to: its system shade, else its family, Other folded into Baseline."""
return shade or (Constants.baseline if type_name == Constants.other else type_name)
@functools.cache
def _recent_additions() -> dict[str, str]:
"""The methods the latest releases added, as name -> when (from the version history)."""
releases, _ = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT)
return {name: f"New in {r.version}, {r.when}" for name, r in _added_entrants(releases)[:_NEW_RELEASES]}
#: The families in the order the legend names them, as (family, short name, full name); Baseline and
#: Other share a colour, so one entry. The system shades follow, by _SHADE_SHORT.
_LEGEND_FAMILIES = [
(Constants.foundational, "Foundation", "Foundation Model"),
(Constants.neural_network, "Neural", "Neural Network"),
(Constants.tree, "Tree", "Tree-based"),
(Constants.baseline, "Baseline", "Baseline / Other"),
(Constants.system, "System", "System"),
]
_SHADE_SHORT = {"closed-source-api": "API", "with-llm": "LLM"}
def _method_legend(rows_meta: Iterable[dict]) -> str:
"""The colours of the names, a line under the Method header (HTML).
Each family or system shade a row on screen carries, by a short name in its colour, the full
name on hover, so the key fits the column and the table needs no legend row.
"""
present = {_family_key(m["type_name"], m["shade"]) for m in rows_meta}
entries = [
(model_type_color.get(name, "#9e9e9e"), short, full)
for name, short, full in _LEGEND_FAMILIES
if name in present
]
entries += [
(color, _SHADE_SHORT[shade], system_shade_label[shade])
for shade, color in system_shade_color.items()
if shade in present
]
return " ".join(
f''
f"{html.escape(short)}"
for c, short, full in entries
)
#: The Summary's search icon; the box beside it opens when the icon is clicked (main.py).
_SEARCH_ICON = (
''
)
def _overview_find() -> str:
"""The Summary's search, in the Method header's corner: an icon opening a box that keeps the rows
whose name holds its text (main.taOverviewTable)."""
find = html.escape(website_texts.OVERVIEW_FIND, quote=True)
return (
f''
)
def section_head(title: str, line: str = "", *, anchor: str = "", hint: str = "", actions: str = "") -> str:
"""A section's heading: its title, beside it a line saying what the section holds (HTML), and
``actions`` at the right. ``hint`` is the title's hover text, for what the line leaves out.
"""
anchor = f' id="{anchor}"' if anchor else ""
hint = f' title="{html.escape(hint, quote=True)}"' if hint else ""
line = f'{line}' if line else ""
return (
f'
'
f'
{html.escape(title)}{line}
{actions}
'
)
#: Where the selected tasks span two kinds, the two the spec line counts the datasets of.
_TASK_HALVES = {"all": ("classification", "regression"), "classification": ("binary", "multiclass")}
def follow_picker(lb: LBContainer) -> str:
"""The "Follow" row under the leaderboard's heading: the methods picked here are marked in
every figure and table of the page (``main.taToggleFollow``).
The list holds the subset's methods in leaderboard order, each in its family colour, without
the ones the reader excluded. The picks live on the page rather than in this markup, so a new
selection keeps them; the chips after the button list them and drop one on a click.
"""
try:
df = lb.load_df()
except FileNotFoundError:
return ""
excluded = set(lb.excluded_methods())
tags = df["Tags"] if "Tags" in df.columns else pd.Series(None, index=df.index)
items, seen = [], set()
for model, type_name, tag in zip(df["Model"], df["TypeName"], tags, strict=True):
base = parse_model(str(model))[0]
if base in seen or base in excluded:
continue
seen.add(base)
items.append(
f''
)
find = html.escape(website_texts.FOLLOW_FIND, quote=True)
return (
'
'
f''
f"{website_texts.FOLLOW_LABEL}"
f'
{"".join(items)}
'
'
'
)
def lb_spec(lb: LBContainer) -> tuple[str, str]:
"""What the selected leaderboard covers, as the line beside its section's title, and the notes
behind it as the title's hover text.
The number of datasets leads, split in two by task where the selection spans both kinds; the
other filters and the settings follow, named as the selector card's summary names them.
"""
subset = lb.subset
n = lb.n_datasets
parts = [f"{n} {'dataset' if str(n) == '1' else 'datasets'}"]
halves = _TASK_HALVES.get(subset.tasks)
if halves:
counts = [(LBContainer(lb.data_root, replace(subset, tasks=half), "").n_datasets, half) for half in halves]
parts.append(" / ".join(f"{count} {TASK_LABELS[half].lower()}" for count, half in counts if count))
else:
parts.append(TASK_LABELS[subset.tasks].lower())
notes = []
if isinstance(subset, BeyondSubset):
parts += [BEYOND_SUBSET_LABELS[key] for key in subset.filters if key != subset.tasks]
parts.append(website_texts.BEYOND_SELECTOR_TAIL[0])
notes += [BEYOND_SUBSET_NOTE[key] for key in subset.filters]
if str(n).isdigit() and int(n) < BEYOND_FEW_DATASETS:
parts.append(
f''
"⚠️ few datasets"
)
else:
if subset.datasets != "all":
parts.append(DATASET_LABELS[subset.datasets].lower())
categories = entrants_categories(subset.entrants)
names = ["models", *(_plain(SYSTEM_CATEGORY_LABELS[key]) for key in categories)]
parts.append(" + ".join(names) if categories else website_texts.SELECTOR_ALONE.format("models"))
notes_on = website_texts.SELECTOR_SETTING_NOTES
parts.append(notes_on["imputed"][0 if subset.imputation == "yes" else 1])
if subset.splits == "lite":
parts.append(notes_on["lite"][0])
notes += [entrants_note(subset.entrants), DATASET_SIZE_NOTE.get(subset.datasets, "")]
if not lb.include_noncommercial:
parts.append(website_texts.SELECTOR_SETTING_NOTES["noncommercial"][1])
if not lb.include_leakage:
parts.append(website_texts.SELECTOR_SETTING_NOTES["leakage"][1])
return " · ".join(part for part in parts if part), " ".join(note for note in notes if note)
# --------------------------------------------------------------------------- #
# Agentic guide
# --------------------------------------------------------------------------- #
def _hero_toggle(slug: str, kind: str) -> str:
"""The hidden checkbox whose state shows one of a page hero's dialogs (see main.taOpenDialog).
Every page is in the document at once, so each page has its own.
"""
return f"ta-{slug}-{kind}-toggle"
def open_doc(slug: str, topic: str) -> str:
"""The click handler that opens page ``slug``'s Learn more dialog on ``topic``."""
return f"taOpenDoc('{_hero_toggle(slug, 'docs')}', '{topic}')"
# A release that adds an entrant opens with "Add new verified model: , …" (the
# update-leaderboard skill writes it that way); the name ends at the first comma, bracket or stop.
_NEW_ENTRANT = re.compile(
r"Add (?:the first )?(?:new )?(?:verified |unverified )?(?:model|system)s?: (.+?)(?:, | \(|\. |;|\.?$)"
)
def _added_entrants(releases: Sequence[Release]) -> list[tuple[str, Release]]:
"""The entrants releases add, newest first, each with the release that added it."""
added = []
for release in releases:
first = re.sub(r"\s+", " ", release.source).split("* ", 1)[-1]
match = _NEW_ENTRANT.match(first)
if match:
added.append((match.group(1), release))
return added
def _newest_entrant(releases: Sequence[Release]) -> str:
"""The entrant the newest release that adds one names, or "" when none does."""
added = _added_entrants(releases)
return added[0][0] if added else ""
def _entrant_facts(df: pd.DataFrame) -> dict[str, str]:
"""How many models and systems a leaderboard lists, a method counting once whatever its variants."""
method = df["Model"].map(lambda m: parse_model(m)[0])
if "MethodClass" in df.columns:
is_system = df["MethodClass"].eq("system")
else:
is_system = df["TypeName"].eq(Constants.system)
if "Verified" in df.columns:
verified = df["Verified"].astype(str).str.strip().eq("✔️")
else:
verified = pd.Series(False, index=df.index)
n_models = method[~is_system].nunique()
n_systems = method[is_system].nunique()
return {
"models": str(n_models),
"methods": str(n_models + n_systems),
"methods_label": "models & systems" if n_systems else "models",
"entrants": f"{n_models} models" + (f" and {n_systems} systems" if n_systems else ""),
"verified": str(method[~is_system & verified].nunique()),
}
def hero_facts(data_root: Path) -> dict[str, str]:
"""The numbers the TabArena hero quotes, read from the published data and the version history.
Methods are counted in the widest entrant pool, so the hero names everything TabArena covers
rather than whichever pool the page opens on.
"""
df = LBContainer(data_root, Subset(entrants=widest_entrants_key()), "").load_df()
releases, _ = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT)
updated = releases[0].when if releases else "—"
newest = _newest_entrant(releases)
if newest:
fresh = website_texts.HERO_FRESH_NEWEST.format(newest=newest)
else:
fresh = website_texts.HERO_FRESH_UPDATED.format(updated=updated)
return {
"datasets": str(LBContainer(data_root, Subset(), "").n_datasets or "—"),
**_entrant_facts(df),
"releases": str(len(releases)),
"since": f"{releases[-1].date:%b %Y}" if releases else "—",
"updated": updated,
"fresh": fresh,
}
def _dialog(toggle: str, label: str, body: str, *, cls: str, head: str = "") -> str:
"""A dialog that shows while its hidden checkbox is checked (see main.taOpenDialog).
``head`` sits above the body and stays in place while the body scrolls.
"""
return (
'
'
f''
'
'
f'
'
''
f'{head}
{body}
'
"
"
)
def citation_html(text_id: str, label: str, bibtex: str) -> str:
"""A benchmark's BibTeX entry with a button that copies it."""
return (
f'
{html.escape(label)}
'
f'
{html.escape(bibtex.strip())}
'
f''
)
def metric_definitions_html() -> str:
"""The leaderboard's columns, defined as the table's header tooltips define them."""
rows = "".join(
f"
'
def learn_more(
toggle: str,
label: str,
topics: Sequence[tuple[str, str, str]],
cta: tuple[str, str] | None = None,
hints: dict[str, str] | None = None,
) -> tuple[str, str]:
"""A page's Learn more row and the dialog it opens, for (key, label, HTML) topics.
Every topic is in the page from the start, so a pill opens at once without a round trip to the
server, and the tabs at the top of the dialog switch topic without closing it. Returned apart,
so the row can sit inside a layout the dialog would otherwise add a gap to. ``hints`` maps a
topic to its pill's hover text. ``cta``, as (label, link), ends the row as a highlighted link out,
which also draws the eye to the pills.
"""
# The tour leads the row (main.taStartTour), where a newcomer looks first.
pills = (
f''
)
for key, name, _ in topics:
hint = (hints or {}).get(key)
title = f' title="{html.escape(hint)}"' if hint else ""
pills += (
f'"
)
if cta:
cta_label, url = cta
pills += (
f'{html.escape(cta_label)}'
'↗\ufe0e'
)
tabs = "".join(
f'"
for key, name, _ in topics
)
panes = "".join(f'{body}' for key, _, body in topics)
row = f'
{website_texts.LEARN_MORE_LABEL}{pills}
'
head = f'
{tabs}
'
return row, _dialog(toggle, label, panes, cls="ta-dialog-docs", head=head)
def _avatar(login: str, size: int) -> str:
"""A maintainer's GitHub avatar, fetched at twice its size for sharp screens."""
return (
f''
)
def _team_dialog(toggle: str, title: str, team: Sequence[tuple[str, str]]) -> str:
"""The maintainers, as (name, GitHub login), each linking to their profile, and who to ask what."""
members = []
for name, login in team:
primary = login == website_texts.TEAM_PRIMARY_CONTACT
badge = f'{website_texts.TEAM_PRIMARY_LABEL}' if primary else ""
members.append(
f'{_avatar(login, 44)}'
f'{html.escape(name)}'
f'@{login}{badge}'
)
body = (
f"
{html.escape(title)}
"
f'
{html.escape(website_texts.TEAM_SUB)}
'
f'
{"".join(members)}
'
f'
{render_markdown(website_texts.TEAM_CONTACT)}
'
)
return _dialog(toggle, title, body, cls="ta-dialog-team")
def _hero_stat(
value: str, label: str, line: str, hint: str, target: str, *, facts: dict[str, str], docs_toggle: str
) -> str:
"""One selling point: a big value, what it counts and a short line, opening its target.
A numeric value counts up whenever the band opens (the ``ta-count-up`` animation in main.py).
"""
value, label, line, hint = (part.format(**facts) for part in (value, label, line, hint))
if value.isdigit():
num = (
f''
f'{value}'
)
else:
num = f'{html.escape(value)}'
inner = (
f'{num}{html.escape(label)}'
f'{html.escape(line)}'
)
title = html.escape(hint)
if target.startswith("doc:"):
action = f"taOpenDoc('{docs_toggle}', '{target[4:]}')"
return f''
if target == "history":
# The version history is the proof that the benchmark is alive, so its line has a live dot.
action = f"taOpenDialog('{VERSION_HISTORY_TOGGLE}')"
return f''
return f'{inner}'
def _page_hero(
slug: str,
facts: dict[str, str],
*,
lead: str,
paper_url: str,
citation: tuple[str, str],
cta: tuple[str, str],
team: tuple[str, Sequence[tuple[str, str]]],
hints: dict[str, str],
stats: Sequence[tuple[str, str, str, str, str]],
docs: Sequence[tuple[str, str, str]],
) -> gr.HTML:
"""The top of a benchmark's page: what it is, its links, and the topics to learn more.
"Why this benchmark?" ends the sentence and opens the selling points (``stats``) under it, hidden at
first; they and the Learn more pills' ``hints`` quote ``facts`` read from the data, so the
numbers cannot drift from what the page shows. The citation (``citation`` is its label and BibTeX), the
team (``team`` is its title and members) and the ``docs`` topics open as dialogs rather than
panels on the page, and ``cta`` ends the Learn more row. The hero takes its name from the page's
link in the top bar and its colour from ``slug``.
"""
name = website_texts.TOPBAR_NAV[slug][1]
cite_toggle, team_toggle, docs_toggle = (_hero_toggle(slug, kind) for kind in ("cite", "team", "docs"))
paper_icon, paper_label = website_texts.HERO_PAPER
cite_icon, cite_label = website_texts.HERO_CITE
head = (
''
f''
f'{paper_icon}{html.escape(paper_label)}'
f''
""
)
avatars = "".join(_avatar(login, 20) for _, login in team[1])
head += (
f'"
)
band = "".join(_hero_stat(*stat, facts=facts, docs_toggle=docs_toggle) for stat in stats)
why_id = f"ta-{slug}-why"
why = (
f' '
)
hints = {key: hint.format(**facts) for key, hint in hints.items()}
learn_row, docs_dialog = learn_more(docs_toggle, f"Learn more about {name}", docs, cta=cta, hints=hints)
cite = f"
'
)
return _dialog(VERSION_HISTORY_TOGGLE, website_texts.VERSION_HISTORY_TITLE, body, cls="ta-dialog-vh")
def make_page_chrome(pages: Sequence[LeaderboardPage]) -> None:
"""The site's bar: benchmark links, the current release, global actions, dialogs.
A rail down the left edge on a wide screen and a bar above the page on a narrow one (main.py's
top-bar CSS), from the same markup.
One block for all of it, since every extra ``gr.HTML`` adds a layout gap. The release shown
and the version-history dialog are read from ``website_texts.VERSION_HISTORY_BUTTON_TEXT``,
so a release recorded there shows up here without a second edit.
"""
releases, note = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT)
gr.HTML(
_top_bar(pages, releases[0] if releases else None)
+ website_texts.COI_HTML
+ _version_history_dialog(releases, note),
apply_default_css=False,
)
# --------------------------------------------------------------------------- #
# BeyondArena components
#
# BeyondArena reuses the per-subset leaderboard table (make_leaderboard) and the
# figure grid (make_figure_grid) unchanged; only the hero's copy, the
# cross-subset overview (an image, not the TabArena HTML heatmap) and the
# per-subset figure set differ (no HPO tuning-trajectory figure — BeyondArena is
# evaluated on a single `core2k` protocol).
# --------------------------------------------------------------------------- #
def beyond_hero_facts(data_root: Path) -> dict[str, str]:
"""The numbers the BeyondArena hero quotes, read from its published data.
The subsets are the selector's filter chips, every axis's "all" aside; they combine freely.
"""
lb = LBContainer(data_root, BeyondSubset(), "")
return {
"datasets": str(lb.n_datasets or "—"),
"subsets": str(len(BEYOND_SUBSET_LABELS)),
**_entrant_facts(lb.load_df()),
}
def make_beyond_hero(data_root: Path, docs: Sequence[tuple[str, str, str]]) -> gr.HTML:
"""The top of the BeyondArena page, quoting :func:`beyond_hero_facts`, with ``docs`` to learn more."""
return _page_hero(
"beyondarena",
beyond_hero_facts(data_root),
lead=website_texts.BEYOND_HERO_LEAD,
paper_url=website_texts.BEYOND_HERO_PAPER_URL,
citation=(website_texts.BEYOND_CITATION_BUTTON_LABEL, website_texts.BEYOND_CITATION_BUTTON_TEXT),
cta=website_texts.BEYOND_HERO_CTA,
team=(website_texts.BEYOND_TEAM_TITLE, website_texts.BEYOND_TEAM),
hints=website_texts.BEYOND_HERO_HINTS,
stats=website_texts.BEYOND_HERO_STATS,
docs=docs,
)
def make_beyond_overview_figure(data_root: Path, include_leakage: bool = True) -> None:
"""The cross-subset overview: best Elo per model / per family across every subset.
Unlike TabArena's HTML heatmap, BeyondArena's overview is the ``plot_subset_results`` image
(``result_plots/per_model_elo`` + ``per_family_elo``). Missing images are skipped gracefully.
``include_leakage`` off reads their twins computed without the methods with a leakage warning
(``result_plots_noleak/``); on, a note names those methods, since the images cannot mark them.
"""
result_dir = Path(data_root) / ("result_plots" if include_leakage else "result_plots" + NO_LEAKAGE_SUFFIX)
images = [
("per_family_elo", "Best Elo per model type (family) across subsets"),
("per_model_elo", "Best Elo per model across subsets"),
]
shown = False
for name, label in images:
if (result_dir / f"{name}.png").exists() or (result_dir / f"{name}.png.zip").exists():
# No fixed height: the per-model figure is about 6:1, and a fixed box letterboxed it.
gr.Image(
unzip_png(result_dir, name),
label=label,
show_label=True,
elem_classes="ta-view-card",
)
shown = True
if not shown:
gr.Markdown("_The cross-subset overview figure is not available yet._", elem_classes="markdown-text")
elif include_leakage:
try:
flagged = leakage_methods(LBContainer(data_root, BeyondSubset(), "").load_df())
except FileNotFoundError:
flagged = []
if flagged:
gr.Markdown(
website_texts.LEAKAGE_OVERVIEW_NOTE.format(names=", ".join(flagged)),
elem_classes=["markdown-text", "ta-lk-note"],
)
'
)
def _team_dialog(toggle: str, title: str, team: Sequence[tuple[str, str]]) -> str:
"""The maintainers, as (name, GitHub login), each linking to their profile, and who to ask what."""
members = []
for name, login in team:
primary = login == website_texts.TEAM_PRIMARY_CONTACT
badge = f'{website_texts.TEAM_PRIMARY_LABEL}' if primary else ""
members.append(
f'{_avatar(login, 44)}'
f'{html.escape(name)}'
f'@{login}{badge}'
)
body = (
f"