Spaces:
Running
Running
Claude Opus 5.5
v0.1.9.12: leakage warning on LimiX-2 with recomputed pools, guided tour, benchmark groups in the rail
3cffdcc | """Gradio component builders for the leaderboard. | |
| Each function creates one piece of the UI inside the surrounding Blocks/render | |
| context. No data-path logic lives here (see ``data_loading.py``) and no copy | |
| lives here (see ``website_texts.py``). | |
| """ | |
| from __future__ import annotations | |
| import datetime | |
| import functools | |
| import html | |
| import json | |
| import math | |
| import os | |
| import re | |
| import textwrap | |
| from collections.abc import Iterable, Sequence | |
| from dataclasses import dataclass, replace | |
| from itertools import groupby | |
| from pathlib import Path | |
| from typing import TYPE_CHECKING | |
| import gradio as gr | |
| import pandas as pd | |
| from markdown_it import MarkdownIt | |
| import website_texts | |
| from constants import ( | |
| Constants, | |
| model_type_color, | |
| model_type_emoji, | |
| system_shade, | |
| system_shade_color, | |
| system_shade_label, | |
| tag_emoji, | |
| type_color, | |
| type_symbols, | |
| variant_color, | |
| ) | |
| from data_loading import ( | |
| AXIS_NOTES, | |
| BEYOND_FEW_DATASETS, | |
| BEYOND_FEW_DATASETS_NOTE, | |
| BEYOND_SUBSET_LABELS, | |
| BEYOND_SUBSET_NOTE, | |
| DATASET_DISTRIBUTION_LABELS, | |
| DATASET_LABELS, | |
| DATASET_SIZE_LABELS, | |
| SYSTEM_CATEGORY_LABELS, | |
| DATASET_SIZE_NOTE, | |
| TASK_LABELS, | |
| BeyondSubset, | |
| LBContainer, | |
| Subset, | |
| commercial_flags, | |
| leakage_datasets, | |
| leakage_evidence, | |
| leakage_methods, | |
| NO_LEAKAGE_SUFFIX, | |
| fits_at_predict_flags, | |
| license_names, | |
| load_leaderboard_csv, | |
| parse_model, | |
| static_url, | |
| entrants_categories, | |
| entrants_note, | |
| unzip_png, | |
| widest_entrants_key, | |
| ) | |
| if TYPE_CHECKING: | |
| from pages import LeaderboardPage | |
| # --------------------------------------------------------------------------- # | |
| # Full per-subset leaderboard table | |
| # --------------------------------------------------------------------------- # | |
| _IMPUTED_INFO = ( | |
| "We impute the performance for models that cannot run on all datasets due to" | |
| " task or dataset size constraints. We impute with the performance of a" | |
| " default RandomForest. We add a postfix [X% IMPUTED] to the model if any" | |
| " results were imputed. The X% shows the percentage of datasets that were" | |
| " imputed. In general, imputation negatively represents the model" | |
| " performance, punishing the model for not being able to run on all datasets." | |
| ) | |
| # Variants a model is evaluated in; all selected by default, and any combination is | |
| # valid. Systems carry no variant tag of their own but tune and ensemble internally, so | |
| # they belong with those. | |
| VARIANT_VALUES = ["default", "tuned", "tuned + ensembled"] | |
| # Columns that drive the filters or are folded into another cell; kept on the frame, | |
| # never rendered as their own column. | |
| _INTERNAL_COLUMNS = ["TypeName", "RefModel", "Imputed", "_variant", "_base", "_search"] | |
| # Shown as "(+115/-110)" after the Elo value instead of as a column of its own. | |
| _CI_COLUMN = "Elo 95% CI" | |
| _ELO_COLUMN = "Elo [⬆️]" | |
| # Rendered as a ✔️ badge on the model name, the way the overview does it. | |
| _VERIFIED_COLUMN = "Verified" | |
| # Always shown, so they are not offered in the column picker. | |
| _ALWAYS_SHOWN = ["#", "Type", "Model"] | |
| # Column header -> the metric-reference entry whose text becomes its hover hint, so | |
| # the tooltips and the documented definitions cannot drift apart. | |
| _LB_TOOLTIP_METRIC = { | |
| _ELO_COLUMN: "🏆 Elo (ranking aggregation)", | |
| "Score [⬆️]": "📊 Score", | |
| "Improvability (%) [⬇️]": "📉 Improvability (%)", | |
| "Rank [⬇️]": "🔢 Average Rank", | |
| "Harmonic Rank [⬇️]": "🎯 Harmonic Rank", | |
| "Median Train Time (s/1K) [⬇️]": "⏱️ Train / Predict Time (s/1K)", | |
| "Median Predict Time (s/1K) [⬇️]": "⏱️ Train / Predict Time (s/1K)", | |
| "Imputed (%) [⬇️]": "🧩 Imputed (%)", | |
| } | |
| _LB_FIXED_TOOLTIPS = { | |
| "#": "Position in this subset's Elo ranking, as published.", | |
| "Type": "Model family; see the legend above the table.", | |
| "Model": ( | |
| "The model, its configuration variant in brackets, and ✔️ when the " | |
| "implementation was verified by its authors or the maintainers. Links to the " | |
| "implementation." | |
| ), | |
| "Hardware": "The hardware the reported runtimes were measured on.", | |
| } | |
| # Which direction is good, per column, for the per-column heatmap. Columns absent | |
| # from this map (#, Hardware) are left unshaded. | |
| _LB_HIGHER_IS_BETTER = { | |
| _ELO_COLUMN: True, | |
| "Score [⬆️]": True, | |
| "Rank [⬇️]": False, | |
| "Harmonic Rank [⬇️]": False, | |
| "Improvability (%) [⬇️]": False, | |
| "Median Train Time (s/1K) [⬇️]": False, | |
| "Median Predict Time (s/1K) [⬇️]": False, | |
| "Imputed (%) [⬇️]": False, | |
| } | |
| # Runtimes span orders of magnitude, so shading them linearly paints every model | |
| # the same green and only the slowest one red. Normalize those in log space. | |
| _LB_LOG_SCALED = { | |
| "Median Train Time (s/1K) [⬇️]", | |
| "Median Predict Time (s/1K) [⬇️]", | |
| } | |
| # The marker the plot explorers put on an imputed method (see the generated | |
| # *_explorer.html); reused here so the same thing looks the same everywhere. | |
| IMPUTED_MARK = "‡" | |
| # Per-column value formatting; everything else falls back to _format_value. | |
| _LB_FORMATS = { | |
| _ELO_COLUMN: lambda v: str(int(round(v))), | |
| "Score [⬆️]": lambda v: f"{v:.3f}", | |
| "Rank [⬇️]": lambda v: f"{v:.2f}", | |
| "Harmonic Rank [⬇️]": lambda v: f"{v:.2f}", | |
| "Improvability (%) [⬇️]": lambda v: f"{v:.2f}", | |
| "Median Train Time (s/1K) [⬇️]": lambda v: f"{v:.2f}", | |
| "Median Predict Time (s/1K) [⬇️]": lambda v: f"{v:.3f}", | |
| "Imputed (%) [⬇️]": lambda v: f"{v:.1f}", | |
| } | |
| def _column_tooltip(column: str) -> str | None: | |
| """Hover hint for a leaderboard column header, or None when there is nothing to add.""" | |
| if column in _LB_FIXED_TOOLTIPS: | |
| return _LB_FIXED_TOOLTIPS[column] | |
| name = _LB_TOOLTIP_METRIC.get(column) | |
| if not name: | |
| return None | |
| for metric in website_texts.METRICS: | |
| if metric["name"] == name: | |
| return f"{metric['details']} · Why we use it: {metric['why']}" | |
| return None | |
| def _format_value(column: str, value) -> str: | |
| """Display string for one cell; `data-sort` keeps the raw value for sorting.""" | |
| if value is None or (isinstance(value, float) and pd.isna(value)): | |
| return "–" | |
| formatter = _LB_FORMATS.get(column) | |
| if formatter and isinstance(value, (int, float)) and not isinstance(value, bool): | |
| return formatter(value) | |
| if isinstance(value, float) and value.is_integer(): | |
| return str(int(value)) | |
| return str(value) | |
| def filter_leaderboard( | |
| df: pd.DataFrame, | |
| *, | |
| models: list[str], | |
| variants: list[str], | |
| show_imputed: bool, | |
| search: str = "", | |
| ) -> pd.DataFrame: | |
| """Apply the table's row filters. | |
| Pure so it can be tested without a Gradio context. `df` carries the internal | |
| ``_base`` / ``_variant`` / ``Imputed`` / ``_search`` columns added by | |
| :func:`_prepare_leaderboard`. `models` is the selected individual models: family | |
| chips are bulk selectors over that list, not a filter of their own. Column | |
| selection is applied when rendering, not here. | |
| """ | |
| sub = df[df["_base"].isin(models)] | |
| # Only narrow when the reader has actually deselected something, so a row whose | |
| # variant we failed to classify can never be dropped silently. | |
| if variants is not None and set(variants) != set(VARIANT_VALUES): | |
| sub = sub[sub["_variant"].isin(variants)] | |
| if not show_imputed: | |
| sub = sub[~sub["Imputed"].astype(bool)] | |
| term = (search or "").strip().lower() | |
| if term: | |
| sub = sub[sub["_search"].str.contains(re.escape(term), regex=True)] | |
| return sub | |
| def _prepare_leaderboard(raw: pd.DataFrame) -> pd.DataFrame: | |
| """Add the internal columns the filters and the renderer need.""" | |
| df = raw.copy() | |
| parsed = [parse_model(str(m)) for m in raw["Model"]] | |
| df["_base"] = [base for base, _, _ in parsed] | |
| # A system has no variant tag of its own, but it tunes and ensembles internally, so the | |
| # variant filter groups it with the tuned ensembles. Keyed off the family rather than the | |
| # name, so this holds for every system and not only AutoGluon. | |
| df["_variant"] = [ | |
| "tuned + ensembled" if not variant and type_name == Constants.system else variant | |
| for (_, variant, _), type_name in zip(parsed, raw["TypeName"], strict=True) | |
| ] | |
| df["_search"] = [ | |
| f"{base} {variant} {type_name}".lower() | |
| for (base, variant, _), type_name in zip(parsed, raw["TypeName"], strict=True) | |
| ] | |
| return df | |
| def leaderboard_families(df: pd.DataFrame) -> list[tuple[str, list[str]]]: | |
| """(family, models) pairs present in `df`, in the legend's family order.""" | |
| families = [] | |
| for type_name in model_type_emoji: | |
| models = sorted(set(df.loc[df["TypeName"] == type_name, "_base"])) | |
| if models: | |
| families.append((type_name, models)) | |
| return families | |
| def fam_chip_colors(families: list[tuple[str, list[str]]]) -> str: | |
| """`--fam` per family, so each chiprow carries its family's colour.""" | |
| rules = [ | |
| f".ta-fam-{re.sub(r'[^a-z0-9]+', '-', type_name.lower())}" | |
| f"{{--fam:{model_type_color.get(type_name, '#9e9e9e')};}}" | |
| for type_name, _ in families | |
| ] | |
| # The variant toggles take their colours from the Leaderboard Overview explorer. | |
| # Generated from the same list the choices come from, so the nth-of-type index | |
| # cannot drift from the option order. | |
| rules += [ | |
| f".ta-variants label:nth-of-type({i + 1}){{--fam:{variant_color[value]};}}" | |
| for i, value in enumerate(VARIANT_VALUES) | |
| if value in variant_color | |
| ] | |
| return "".join(rules) | |
| def _group_handler(index: int, models: list[str], render): | |
| """A model chip group changed: bring its family chip in line, redraw the table.""" | |
| def handler(*values): | |
| selected = values[index] or [] | |
| return gr.update(value=set(selected) == set(models)), render(*values) | |
| return handler | |
| def _family_handler(index: int, models: list[str], render): | |
| """A family chip was toggled: select or clear all of its models, redraw the table.""" | |
| def handler(checked, *values): | |
| updated = list(values) | |
| updated[index] = list(models) if checked else [] | |
| return gr.update(value=updated[index]), render(*updated) | |
| return handler | |
| def _heatmap_bounds(df: pd.DataFrame, columns: list[str]) -> dict[str, tuple[float, float]]: | |
| """Per-column (lo, hi) over the rendered rows, in the scale used for shading.""" | |
| bounds = {} | |
| for column in columns: | |
| if column not in _LB_HIGHER_IS_BETTER or column not in df.columns: | |
| continue | |
| values = pd.to_numeric(df[column], errors="coerce").dropna() | |
| if column in _LB_LOG_SCALED: | |
| values = values[values > 0] | |
| values = values.apply(math.log10) | |
| if len(values) < 2 or values.min() == values.max(): | |
| continue | |
| bounds[column] = (float(values.min()), float(values.max())) | |
| return bounds | |
| def _heatmap_style(column: str, value: float, bounds: dict[str, tuple[float, float]]) -> str: | |
| """Inline background for one shaded cell, or "" when the column is not shaded.""" | |
| if column not in bounds: | |
| return "" | |
| lo, hi = bounds[column] | |
| scaled = value | |
| if column in _LB_LOG_SCALED: | |
| if value <= 0: | |
| return "" | |
| scaled = math.log10(value) | |
| frac_best = (scaled - lo) / (hi - lo) | |
| if not _LB_HIGHER_IS_BETTER[column]: | |
| frac_best = 1 - frac_best | |
| frac_best = max(0.0, min(1.0, frac_best)) | |
| return f' style="{_heat_style(1 - frac_best)}"' | |
| def _model_cell(row: pd.Series) -> str: | |
| """The Model cell, styled like the cross-subset overview's.""" | |
| color = type_color(row["TypeName"], system_shade(row["TypeName"], row.get("Tags"))) | |
| _, variant, url = parse_model(str(row["Model"])) | |
| name = html.escape(row["_base"]) | |
| if variant: | |
| name += f' <span class="ta-variant">({html.escape(variant)})</span>' | |
| if str(row.get(_VERIFIED_COLUMN, "")).strip() == "✔️": | |
| name += ' <span class="ta-verified" title="Verified implementation">✔️</span>' | |
| if bool(row.get("Imputed", False)): | |
| name += ( | |
| f' <span class="ta-imp" title="Some results are imputed">{IMPUTED_MARK}</span>' | |
| ) | |
| if url: | |
| return ( | |
| f'<a href="{html.escape(url, quote=True)}" target="_blank" rel="noopener" ' | |
| f'style="color:{color};font-weight:600;">{name}' | |
| f'<span class="ta-link-icon">↗</span></a>' | |
| ) | |
| return f'<span style="color:{color};font-weight:600;">{name}</span>' | |
| def leaderboard_table_html(df: pd.DataFrame, columns: list[str], table_id: str) -> str: | |
| """Render the leaderboard as a sortable HTML table in the overview's style. | |
| Every header is clickable (``taSortTable`` in ``main.py``'s head) and carries the | |
| column's definition as a hover hint. Numeric cells put the raw value in | |
| ``data-sort`` and the formatted one in the text, which is what lets the Elo cell | |
| show its confidence interval without that text taking part in the sort. | |
| """ | |
| metric_columns = [c for c in columns if c not in _ALWAYS_SHOWN] | |
| header = [ | |
| '<th class="ta-th-sort ta-th-info" data-type="num" title="' | |
| + html.escape(_LB_FIXED_TOOLTIPS["#"], quote=True) | |
| + '">#</th>', | |
| '<th class="ta-th-sort ta-th-info" title="' | |
| + html.escape(_LB_FIXED_TOOLTIPS["Type"], quote=True) | |
| + '">Type</th>', | |
| '<th class="ta-th-sort ta-th-info ta-th-left" title="' | |
| + html.escape(_LB_FIXED_TOOLTIPS["Model"], quote=True) | |
| + '">Model</th>', | |
| ] | |
| for column in metric_columns: | |
| tooltip = _column_tooltip(column) | |
| label = html.escape(column) | |
| marker = "" | |
| if column == _ELO_COLUMN: | |
| label += ' <span class="ta-ci-head">(95% CI)</span>' | |
| # Tells taExportTable to split the interval into its own CSV column. | |
| marker = ' data-col="elo"' | |
| classes = "ta-th-sort" + (" ta-th-info" if tooltip else "") | |
| title = f' title="{html.escape(tooltip, quote=True)}"' if tooltip else "" | |
| header.append( | |
| f'<th class="{classes}" data-type="num"{marker}{title}>{label}</th>' | |
| ) | |
| # No medals, unlike the cross-subset overview: this table is the full ranking and | |
| # already carries the # column, so podium marks only added a competing ordering. | |
| bounds = _heatmap_bounds(df, metric_columns) | |
| body = [] | |
| for _, row in df.iterrows(): | |
| shade = system_shade(row["TypeName"], row.get("Tags")) | |
| color = type_color(row["TypeName"], shade) | |
| cells = [ | |
| f'<td class="ta-num" data-sort="{row["#"]}">{row["#"]}</td>', | |
| f'<td class="ta-type-cell" data-sort="{html.escape(str(row["TypeName"]), quote=True)}">' | |
| f'<span class="ta-pill" style="background:{color}22;color:{color};' | |
| f'border:1px solid {color}66;">{type_symbols(row["Type"], shade)}</span></td>', | |
| # data-export="text": the CSV should carry the displayed name, not the | |
| # lowercased key this cell sorts on. | |
| f'<td class="ta-model-cell" data-export="text" ' | |
| f'data-sort="{html.escape(row["_base"].lower(), quote=True)}">' | |
| f"{_model_cell(row)}</td>", | |
| ] | |
| for column in metric_columns: | |
| value = row.get(column) | |
| text = _format_value(column, value) | |
| if text == "–": | |
| cells.append('<td class="ta-na">–</td>') | |
| continue | |
| numeric = isinstance(value, (int, float)) and not isinstance(value, bool) | |
| shade = _heatmap_style(column, float(value), bounds) if numeric else "" | |
| if column == _ELO_COLUMN: | |
| ci = row.get(_CI_COLUMN) | |
| interval, ci_attr = "", "" | |
| if ci is not None and not pd.isna(ci): | |
| interval = f' <span class="ta-ci">({html.escape(str(ci))})</span>' | |
| ci_attr = f' data-ci="{html.escape(str(ci), quote=True)}"' | |
| cells.append( | |
| f'<td class="ta-num" data-sort="{value}"{ci_attr}{shade}>' | |
| f"{text}{interval}</td>" | |
| ) | |
| continue | |
| sort_key = value if numeric else html.escape(str(value), quote=True) | |
| cells.append(f'<td class="ta-num" data-sort="{sort_key}"{shade}>{text}</td>') | |
| body.append(f"<tr>{''.join(cells)}</tr>") | |
| caption = f"{len(df)} row{'' if len(df) == 1 else 's'}" | |
| shaded = "green is better, red is worse, per column" if bounds else "unshaded" | |
| return ( | |
| f'<div class="ta-scroll"><table class="ta-overview ta-lbtable" id="{table_id}">' | |
| f"<thead><tr>{''.join(header)}</tr></thead>" | |
| f"<tbody>{''.join(body)}</tbody></table></div>" | |
| f'<p class="ta-cap">{caption} · click a column header to sort, hover one for its ' | |
| f"definition · {shaded} · {IMPUTED_MARK} marks a model with imputed results</p>" | |
| ) | |
| def make_leaderboard(lb: LBContainer, *, opened: bool = False) -> None: | |
| """The full leaderboard table for one subset, as a card like the figure tiles, ``opened`` or | |
| closed. | |
| The table is a generated artifact (`leaderboard_table.html`, built by | |
| `tabarena.plot.interactive.leaderboard_table`), embedded the same way as the | |
| other interactive plots. It lives upstream so that it reuses the explorers' | |
| family and variant colours, chip components and imputation markers rather than | |
| reimplementing them here, where they drifted. | |
| Its button shows and hides it in the browser (``main.taToggleTable``); its frame loads only | |
| while it is shown. The download sits in the card's header but is the | |
| frame's own (``main.taLeaderboardCsv``): only the frame knows the current filters, columns and | |
| sort order. On TabArena the frame is also told which methods the latest releases added, for | |
| its "new" badges (``data-new``, ``main.taSendNew``). | |
| Subsets whose artifacts predate it fall back to :func:`make_leaderboard_gradio`, | |
| this app's own table. | |
| """ | |
| url = lb.html_url("leaderboard_table") | |
| if url is None: | |
| make_leaderboard_gradio(lb) | |
| return | |
| uid = _panel_uid(lb, "leaderboard_table") | |
| fresh = _recent_additions() if isinstance(lb.subset, Subset) else {} | |
| attrs = f'data-new="{html.escape(json.dumps(fresh), quote=True)}" ' if fresh else "" | |
| frame = _explorer_frame( | |
| url, f"Full leaderboard table, {lb.name}", height=1000, extra_attrs=attrs, exclude=lb.excluded_methods() | |
| ) | |
| caption = html.escape(website_texts.FIGURE_CAPTIONS["table"], quote=True) | |
| gr.HTML( | |
| f'<section class="ta-tile ta-lbtile" id="{uid}" data-role="table"><div class="ta-figbar">' | |
| f'<span class="ta-figbar-title" title="{caption}">{website_texts.TABLE_TITLE}</span>' | |
| f'<span class="ta-figbar-sub" title="{caption}">{caption}</span><span class="ta-figbar-actions">' | |
| f'<button type="button" class="ta-viewbtn ta-csvbtn"{"" if opened else " hidden"} onclick="taLeaderboardCsv(this)" ' | |
| 'title="Download the rows and columns shown, in the current sort order">Download CSV</button>' | |
| f'<button type="button" class="ta-viewbtn ta-tabletoggle" aria-expanded="{str(opened).lower()}" ' | |
| f'data-show="{website_texts.TABLE_SHOW}" data-hide="{website_texts.TABLE_HIDE}" ' | |
| f'onclick="taToggleTable(this)">{website_texts.TABLE_HIDE if opened else website_texts.TABLE_SHOW}</button>' | |
| "</span></div>" | |
| f'<div class="ta-figview ta-lbtile-body"{"" if opened else " hidden"}>{frame}</div></section>', | |
| padding=False, | |
| apply_default_css=False, | |
| elem_classes="ta-flush", | |
| ) | |
| def make_leaderboard_gradio(lb: LBContainer) -> None: | |
| """The fallback table, rendered by this app rather than embedded. | |
| Kept for subsets whose artifacts were generated before `leaderboard_table.html` | |
| existed. :func:`make_leaderboard` is the current path; prefer fixing the | |
| upstream generator over this. | |
| Replaces the third-party `gradio_leaderboard` widget (which pinned Gradio < 6) | |
| with the same hand-rolled HTML table the cross-subset overview uses: type pills, | |
| dotted-underline model links, tooltip-carrying headers, sticky header and scroll | |
| box. Model selection mirrors the plot explorers' edit view — a family chip above | |
| its models, in the family's colour. Sorting and CSV export are client-side | |
| (`taSortTable` / `taExportTable`); the filters round-trip to rebuild the HTML. | |
| """ | |
| raw = lb.load_df() | |
| has_imputed = bool(raw["Imputed"].any()) | |
| df = _prepare_leaderboard(raw) | |
| if not has_imputed: | |
| df = df.drop(columns=["Imputed (%) [⬇️]"]) | |
| # The CI and Verified columns are folded into the Elo and Model cells. | |
| renderable = [ | |
| c | |
| for c in df.columns | |
| if c not in _INTERNAL_COLUMNS + [_CI_COLUMN, _VERIFIED_COLUMN] | |
| ] | |
| optional = [c for c in renderable if c not in _ALWAYS_SHOWN] | |
| families = leaderboard_families(df) | |
| table_id = f"lb-{lb.subset.rel_path.replace('/', '-')}" | |
| def render(*values) -> str: | |
| """Redraw the table from the controls' values, positional as Gradio passes them.""" | |
| count = len(families) | |
| models = [m for group in values[:count] for m in (group or [])] | |
| variants, show_imputed, columns, search = values[count : count + 4] | |
| sub = filter_leaderboard( | |
| df, | |
| models=models, | |
| variants=variants, | |
| show_imputed=show_imputed or not has_imputed, | |
| search=search, | |
| ) | |
| chosen = [c for c in renderable if c in set(columns or []) or c in _ALWAYS_SHOWN] | |
| return leaderboard_table_html(sub, chosen, table_id) | |
| with gr.Column(elem_classes="ta-lb", elem_id=_panel_uid(lb, "leaderboard_table")): | |
| gr.HTML( | |
| # The table borrows the overview's stylesheet; inject it here rather than | |
| # relying on the legend, which this card no longer draws (the family chips | |
| # below say the same thing, in colour). | |
| _OVERVIEW_CSS | |
| + f"<style>{fam_chip_colors(families)}</style>" | |
| + '<div class="ta-lb-bar">' | |
| '<span class="ta-lb-title">Full leaderboard table</span>' | |
| '<span class="ta-lb-sub">every column sorts · hover a header for what it ' | |
| "means</span>" | |
| # Same green as the figure panels' export controls (.ta-exportbtn). | |
| f'<button class="ta-viewbtn ta-exportbtn ta-csvbtn" ' | |
| f'onclick="taExportTable(\'{table_id}\')" ' | |
| 'title="Download the rows and columns currently shown, in the current sort ' | |
| 'order">⬇️ CSV</button>' | |
| "</div>" | |
| ) | |
| # Model selection, laid out like the Pareto explorer's chiprows: one row per | |
| # family, the family chip toggling all of its models at once. | |
| fam_toggles, model_groups = [], [] | |
| for type_name, models in families: | |
| fam_class = f"ta-fam-{re.sub(r'[^a-z0-9]+', '-', type_name.lower())}" | |
| with gr.Row(elem_classes="ta-chiprow"): | |
| fam_toggles.append( | |
| gr.Checkbox( | |
| value=True, | |
| label=f"{model_type_emoji.get(type_name, '')} {type_name} ×{len(models)}", | |
| show_label=False, | |
| container=False, | |
| interactive=True, | |
| elem_classes=["ta-famchip", fam_class], | |
| scale=0, | |
| min_width=210, | |
| ) | |
| ) | |
| model_groups.append( | |
| gr.CheckboxGroup( | |
| choices=models, | |
| value=models, | |
| show_label=False, | |
| container=False, | |
| interactive=True, | |
| elem_classes=["ta-chips", fam_class], | |
| scale=1, | |
| ) | |
| ) | |
| with gr.Row(elem_classes="ta-lb-controls"): | |
| variant = gr.CheckboxGroup( | |
| choices=VARIANT_VALUES, | |
| value=VARIANT_VALUES, | |
| label="⚙️ Variants", | |
| interactive=True, | |
| elem_classes=["ta-btns", "ta-variants"], | |
| scale=3, | |
| min_width=290, | |
| ) | |
| show_imputed = gr.Checkbox( | |
| value=True, | |
| label=f"{IMPUTED_MARK} Include imputed", | |
| info=_IMPUTED_INFO, | |
| interactive=True, | |
| visible=has_imputed, | |
| elem_classes=["ta-btns", "ta-btns-imputed"], | |
| scale=1, | |
| min_width=190, | |
| ) | |
| columns = gr.Dropdown( | |
| choices=optional, | |
| value=optional, | |
| multiselect=True, | |
| label="📋 Columns", | |
| info="# / Type / Model always shown", | |
| interactive=True, | |
| scale=2, | |
| min_width=220, | |
| ) | |
| search = gr.Textbox( | |
| label="🔍 Search", | |
| placeholder="model or type…", | |
| interactive=True, | |
| scale=1, | |
| min_width=150, | |
| ) | |
| state = [*model_groups, variant, show_imputed, columns, search] | |
| table = gr.HTML(render(*[c.value for c in state])) | |
| # `.input` rather than `.change`: a family chip rewrites its group's value and | |
| # each group rewrites its family chip, so reacting to programmatic changes too | |
| # would let the two bounce off each other. | |
| for index, (models, group, fam) in enumerate( | |
| zip([m for _, m in families], model_groups, fam_toggles, strict=True) | |
| ): | |
| group.input( | |
| _group_handler(index, models, render), | |
| state, | |
| [fam, table], | |
| api_visibility="private", | |
| ) | |
| fam.input( | |
| _family_handler(index, models, render), | |
| [fam, *state], | |
| [group, table], | |
| api_visibility="private", | |
| ) | |
| for control in (variant, show_imputed, columns, search): | |
| control.input(render, state, table, api_visibility="private") | |
| # --------------------------------------------------------------------------- # | |
| # Per-subset figures | |
| # --------------------------------------------------------------------------- # | |
| def _explorer_frame( | |
| url: str, | |
| title: str, | |
| height: int = 720, | |
| extra_attrs: str = "", | |
| exclude: list[str] | None = None, | |
| ) -> str: | |
| """A sandboxed frame for one interactive explorer page, loaded from the page's static URL. | |
| The page is a file Gradio serves (``gr.set_static_paths`` in ``main``), so a render sends this | |
| tag rather than the page itself, and the browser keeps the page cached between renders. The URL | |
| goes in ``data-src``: ``main.taLoadFrames`` sets ``src`` once the frame nears the screen, with | |
| the site's theme in the URL's hash, so a frame further down the page or in a closed accordion | |
| costs nothing until it is read. | |
| The sandbox runs the page's scripts and lets it download its exports, nothing else: without | |
| ``allow-same-origin`` the frame stays cross-origin, so the host talks to it over postMessage. | |
| The explorer posts its content height and the listener in ``main.py``'s head sizes the frame | |
| to fit (``height`` is the placeholder until then); paper view, exports, the metric, the time | |
| axis and the exclusions travel the other way (``main.taPaperView``, ``main.taExport``, | |
| ``main.taSendMetric`` and its siblings). | |
| ``extra_attrs`` is spliced into the tag for frames the host configures once they are up (a | |
| Pareto tile's time axis, the per-dataset browser's opening filters). ``exclude`` names the | |
| methods the frame should drop (see ``main.taSendExclude``), as a ``data-exclude`` JSON attribute. | |
| """ | |
| if exclude: | |
| extra_attrs += f' data-exclude="{html.escape(json.dumps(sorted(exclude)), quote=True)}"' | |
| return ( | |
| f'<iframe data-src="{html.escape(url, quote=True)}" class="ta-explorer" ' | |
| f'style="width:100%;height:{height}px;border:0;background:transparent" ' | |
| f'sandbox="allow-scripts allow-downloads" {extra_attrs}' | |
| f'title="{html.escape(title, quote=True)}"></iframe>' | |
| ) | |
| def _panel_uid(lb: LBContainer, key: str) -> str: | |
| """A DOM-safe id unique to one (subset, figure) panel.""" | |
| return "ta-fig-" + re.sub(r"[^a-z0-9]+", "-", f"{lb.subset.rel_path}-{key}".lower()).strip("-") | |
| def _download_menu(uid: str, label: str = "Download") -> str: | |
| """A panel's download button, opening the formats its figure exports to (main.taExport).""" | |
| formats = "".join( | |
| f"<button type=\"button\" onclick=\"taExport('{uid}','{fmt}')\">{fmt.upper()}</button>" | |
| for fmt in ("svg", "pdf", "png") | |
| ) | |
| return ( | |
| f'<details class="ta-dl" id="{uid}-export"><summary class="ta-viewbtn">{html.escape(label)}</summary>' | |
| f'<div class="ta-dl-menu">{formats}</div></details>' | |
| ) | |
| # What the reader is optimizing for -> which figure answers it first, and (for the Pareto | |
| # panels) which time axis they plot against. Every Pareto explorer is generated from the same | |
| # points upstream, so switching axis is a different view, not a different computation. | |
| # Insertion order = selector order; first = default. | |
| # Prediction speed before training speed: it is what serving costs, so it is the constraint | |
| # more readers arrive with. End-to-end time is both together. | |
| CARE_LABELS = { | |
| "quality": "🏆 Best quality", | |
| "infer": "⚡ Fast predictions", | |
| "train": "⏱️ Fast to train", | |
| "total": "🏁 Fast end-to-end", | |
| } | |
| # The second half of "I care about": which headline metric leads. Elo and Improvability | |
| # answer different questions, and the leaderboard has always reported both. | |
| METRIC_LABELS = { | |
| "elo": "🏅 Consistent wins", | |
| "imp": "📉 Relative gains", | |
| } | |
| METRIC_NOTES = { | |
| "elo": ( | |
| "Ranks by Elo: how reliably a method beats the others head to head, whatever the margin." | |
| ), | |
| "imp": ( | |
| "Ranks by Improvability: how far a method sits from the best one on each dataset, in percent." | |
| ), | |
| } | |
| CARE_NOTES = { | |
| "quality": "Ranked purely on accuracy. Start here if compute is not your constraint.", | |
| "infer": "Leads with the accuracy-versus-prediction-time trade-off, which is what serving costs.", | |
| "train": "Leads with the accuracy-versus-training-time trade-off.", | |
| "total": "Leads with the accuracy-versus-end-to-end-time trade-off: training and prediction together.", | |
| } | |
| def has_end_to_end(lb: LBContainer) -> bool: | |
| """Whether the subset's Pareto explorer offers end-to-end (train + inference) time. | |
| Explorers generated before that axis existed do not (BeyondArena's, for now), and no | |
| end-to-end panel is shown for them. | |
| """ | |
| return _offers_x_total(str(lb.base_path / "pareto_front_explorer.html")) | |
| def _offers_x_total(path: str) -> bool: | |
| try: | |
| return '"key": "x_total"' in Path(path).read_text(encoding="utf-8") | |
| except FileNotFoundError: | |
| return False | |
| class _Tile: | |
| """One figure of the grid under "Your leaderboard". | |
| ``role`` decides which headline metric the tile shows: ``"overview"`` and ``"scatter"`` tiles | |
| take one each (see ``main.taTileMetric``), ``"matrix"`` has none. ``html_name`` is the explorer | |
| page and ``img_name`` the static figure a subset may ship instead; ``xaxis`` is the time axis | |
| a Pareto explorer opens on. | |
| """ | |
| key: str | |
| role: str | |
| title: str | |
| html_name: str | |
| img_name: str | |
| xaxis: str = "" | |
| wide: bool = False | |
| #: Every figure the grid can hold, in its order when the reader cares about quality; "I care about" | |
| #: moves another one to the front in the browser (the `.ta-page[data-care]` rules in main.py), so | |
| #: no choice there costs a render. Inference and end-to-end time are one explorer page, which | |
| #: offers every time axis; each tile's frame is told which one to open on (`main.taSendXAxis`). | |
| _TILES = ( | |
| _Tile( | |
| "overview", | |
| "overview", | |
| "Leaderboard Overview", | |
| "leaderboard_overview_explorer", | |
| "tuning-impact-elo", | |
| wide=True, | |
| ), | |
| _Tile( | |
| "infer", | |
| "scatter", | |
| "Pareto Front, inference time", | |
| "pareto_front_explorer", | |
| "pareto_front_improvability_vs_time_infer", | |
| xaxis="x_infer", | |
| ), | |
| _Tile("total", "scatter", "Pareto Front, end-to-end time", "pareto_front_explorer", "", xaxis="x_total"), | |
| _Tile( | |
| "train", | |
| "scatter", | |
| "Pareto Front, train time", | |
| "pareto_front_explorer", | |
| "pareto_front_improvability_vs_time_train", | |
| xaxis="x_train", | |
| ), | |
| _Tile("trajectories", "scatter", "Tuning Trajectories", "tuning_trajectories_explorer", "pareto_n_configs_imp"), | |
| _Tile("winrate", "matrix", "Win-rate Matrix", "winrate_explorer", "winrate_matrix", wide=True), | |
| ) | |
| #: The figures of each view that shows them, by their ``_TILES`` keys. | |
| VIEW_TILES = { | |
| "leaderboard": ("overview",), | |
| "time": ("infer", "total", "train", "trajectories"), | |
| "h2h": ("winrate",), | |
| } | |
| def _figure_tiles(lb: LBContainer) -> list[_Tile]: | |
| """The tiles this subset has a figure for. | |
| A tile shows when its explorer or its static figure ships, and the end-to-end one only where | |
| the Pareto explorer offers that axis. The scatter tiles pair up two to a row; an odd one out | |
| spans the row, so the grid has no hole whatever "I care about" puts first. | |
| """ | |
| tiles = [] | |
| for tile in _TILES: | |
| if tile.key == "total" and not has_end_to_end(lb): | |
| continue | |
| if lb.html_url(tile.html_name) is None and not (tile.img_name and lb.has_image(tile.img_name)): | |
| continue | |
| tiles.append(tile) | |
| scatter = [tile for tile in tiles if tile.role == "scatter"] | |
| if len(scatter) % 2: | |
| tiles[tiles.index(scatter[-1])] = replace(scatter[-1], wide=True) | |
| return tiles | |
| def _tile_html(lb: LBContainer, tile: _Tile) -> str: | |
| """One tile: a header row (title, caption, actions) above the figure. | |
| The figure is the explorer where the subset ships one, with a switch to the static figure | |
| when that ships too (``main.taSwitchView``); a subset whose artifacts predate the explorers | |
| shows the static figure alone. The caption shows only where it fits beside the title whole | |
| (``main.taFitCaptions``); the title's hover text carries it either way. | |
| """ | |
| uid = _panel_uid(lb, tile.key) | |
| url = lb.html_url(tile.html_name) | |
| image = lb.image_path(tile.img_name) if tile.img_name and lb.has_image(tile.img_name) else None | |
| actions = "" | |
| if url and image: | |
| actions += ( | |
| f'<button type="button" class="ta-viewbtn" id="{uid}-btn" aria-pressed="false" ' | |
| f"onclick=\"taSwitchView('{uid}')\">Static figure</button>" | |
| ) | |
| if url: | |
| # The tile opens in paper view, the figure alone; this opens its controls, and | |
| # `aria-pressed` tracks whether paper view is on. The downloads show in paper view, the | |
| # figure an export is made from. | |
| actions += ( | |
| f'<button type="button" class="ta-viewbtn" id="{uid}-paper" aria-pressed="true" ' | |
| f"onclick=\"taPaperView('{uid}')\">Customize</button>" + _download_menu(uid) | |
| ) | |
| expand = html.escape(website_texts.TILE_EXPAND, quote=True) | |
| actions += ( | |
| f'<button type="button" class="ta-viewbtn ta-expand" aria-pressed="false" title="{expand}" ' | |
| f'aria-label="{expand}" ' | |
| f"onclick=\"taExpandTile('{uid}')\">⤢</button>" | |
| ) | |
| caption = html.escape(website_texts.FIGURE_CAPTIONS[tile.key], quote=True) | |
| views = "" | |
| alt = html.escape(f"{tile.title}, {lb.name}", quote=True) | |
| if url: | |
| frame = _explorer_frame( | |
| url, | |
| f"{tile.title} [{lb.name}]", | |
| height=320 if tile.key == "overview" else 800 if tile.key == "winrate" else 500, | |
| extra_attrs=f'data-xaxis="{tile.xaxis}" ' if tile.xaxis else "", | |
| exclude=lb.excluded_methods(), | |
| ) | |
| views += f'<div class="ta-figview" id="{uid}-i">{frame}</div>' | |
| if image: | |
| hidden = " ta-hidden" if url else "" | |
| views += ( | |
| f'<div class="ta-figview ta-figimg{hidden}" id="{uid}-s">' | |
| f'<img src="{static_url(Path(image))}" alt="{alt}" loading="lazy"></div>' | |
| ) | |
| return ( | |
| f'<section class="ta-tile{" ta-tile-wide" if tile.wide else ""}" id="{uid}" ' | |
| f'data-key="{tile.key}" data-role="{tile.role}">' | |
| f'<div class="ta-figbar"><span class="ta-figbar-title" title="{caption}">{html.escape(tile.title)}</span>' | |
| f'<span class="ta-figbar-sub" title="{caption}">{caption}</span>' | |
| f'<span class="ta-figbar-actions">{actions}</span></div>{views}</section>' | |
| ) | |
| def make_figure_grid(lb: LBContainer, keys: Sequence[str]) -> None: | |
| """The subset's figures of one view, by their ``_TILES`` keys, as one grid: the overview and | |
| the win rates span the width, the scatter plots (the three Pareto fronts and the tuning | |
| trajectories) sit two to a row. | |
| One HTML block, so "I care about" and "Measured by" reach it without a render: the page | |
| carries the reader's choice as ``data-care`` / ``data-metric`` (``main.taSelector`` writes | |
| them), the stylesheet orders the tiles from those, and ``main.taSendMetric`` tells each | |
| explorer which metric to open on. | |
| """ | |
| tiles = [tile for tile in _figure_tiles(lb) if tile.key in keys] | |
| no_total = "" if any(tile.key == "total" for tile in tiles) else " ta-no-total" | |
| gr.HTML( | |
| f'<div class="ta-figgrid{no_total}">' + "".join(_tile_html(lb, tile) for tile in tiles) + "</div>", | |
| padding=False, | |
| apply_default_css=False, | |
| elem_classes="ta-flush", | |
| ) | |
| #: The dataset browser's icon: a list of datasets. | |
| _PD_ICON = ( | |
| '<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.9" ' | |
| 'stroke-linecap="round"><path d="M7.5 5h9M7.5 10h9M7.5 15h9"/>' | |
| '<g fill="currentColor" stroke="none"><circle cx="3.6" cy="5" r="1.4"/><circle cx="3.6" cy="10" r="1.4"/>' | |
| '<circle cx="3.6" cy="15" r="1.4"/></g></svg>' | |
| ) | |
| def make_per_dataset_block( | |
| lb: LBContainer, | |
| *, | |
| source_subset: Subset | BeyondSubset | None = None, | |
| filter_attrs: str | None = None, | |
| ) -> None: | |
| """The per-dataset browser, the page's Datasets view: a panel like the figure tiles, open. | |
| The artifact is published once per (entrants, imputation, splits) rather than for all 60 | |
| task/dataset cells: a dataset's own numbers do not depend on which *other* datasets share | |
| its leaderboard, so the browser is read from the unrestricted cell and told which slice the | |
| reader currently has selected. It filters itself from there, and the reader can widen it | |
| again from inside without leaving the page. Its frame loads only once the view is shown | |
| (main.taLoadFrames), since the page it embeds carries every dataset's results and is the | |
| largest artifact the site ships. | |
| ``source_subset`` and ``filter_attrs`` are for BeyondArena, whose cells have no | |
| ``tasks="all", datasets="all"`` sibling: it names the cell that carries the browser and the | |
| ``data-`` attributes that tell the frame which slice is selected. | |
| """ | |
| uid = _panel_uid(lb, "per_dataset_explorer") | |
| source = LBContainer( | |
| data_root=lb.data_root, | |
| subset=source_subset or replace(lb.subset, tasks="all", datasets="all"), | |
| name=lb.name, | |
| ) | |
| if filter_attrs is None: | |
| # The dataset axis carries either a size or a distribution; the browser has a filter for | |
| # each, so the value goes to the one it belongs to. | |
| filter_attrs = ( | |
| f'data-task="{html.escape(lb.subset.tasks, quote=True)}" ' | |
| f'data-size="{lb.subset.datasets if lb.subset.datasets in DATASET_SIZE_LABELS else "all"}" ' | |
| f'data-balance="{lb.subset.datasets if lb.subset.datasets in DATASET_DISTRIBUTION_LABELS else "all"}" ' | |
| ) | |
| url = source.html_url("per_dataset_explorer") | |
| teaser = html.escape(website_texts.PER_DATASET_TEASER, quote=True) | |
| n = source.n_datasets | |
| caption = website_texts.PER_DATASET_CAPTION.format(n=n) if n else website_texts.FIGURE_CAPTIONS["per_dataset"] | |
| with gr.Column(elem_classes="ta-figpanel", elem_id=uid): | |
| gr.HTML( | |
| f'<div class="ta-figbar"><span class="ta-figbar-title" title="{teaser}">' | |
| f"{website_texts.PER_DATASET_TITLE}</span>" | |
| f'<span class="ta-figbar-sub">{html.escape(caption)}</span>' | |
| f'<span class="ta-figbar-actions">{_download_menu(uid, "Download the chart") if url else ""}</span></div>' | |
| ) | |
| if url is None: | |
| gr.HTML(f'<p class="ta-pd-teaser">{website_texts.PER_DATASET_MISSING}</p>') | |
| return | |
| # `-i` matches what `taExport` and `taPaperView` look for on the other panels. | |
| with gr.Column(elem_id=f"{uid}-i", elem_classes=["ta-figview", "ta-figview-flush"]): | |
| gr.HTML( | |
| _explorer_frame( | |
| url, | |
| f"Per-dataset results, {lb.name}", | |
| height=1020, | |
| # Read back by `main.taSendPerDatasetFilter` once the frame reports its | |
| # height, so the browser opens on the slice selected above it. | |
| extra_attrs=filter_attrs, | |
| exclude=lb.excluded_methods(), | |
| ) | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Cross-subset overview (Elo heatmap, rendered as HTML for links + grouping) | |
| # --------------------------------------------------------------------------- # | |
| # Columns of the overview: (label, group, subset, value_column). Always imputation=yes / | |
| # splits=all. `value_column` is None for the subset columns, which report whichever metric the | |
| # selector picked; the two cost columns pin their own instead, so the table reads as | |
| # "how good, and what it costs" before it fans out per subset. | |
| _TRAIN_TIME_COLUMN = "Median Train Time (s/1K) [⬇️]" | |
| _PREDICT_TIME_COLUMN = "Median Predict Time (s/1K) [⬇️]" | |
| #: Hover text of the † on a fit time that includes the prediction call (`FitsAtPredict`). | |
| _FIT_AT_PREDICT_HINT = ( | |
| "This system fits inside its prediction call, so its fit time is the time of that call, " | |
| "from training data to predictions. Its predict time holds the fit too." | |
| ) | |
| #: The dagger's key under the train-time header. | |
| _FIT_AT_PREDICT_KEY = "fit inside predict" | |
| _FIXED_COLUMN_SPECS = { | |
| "Fit (s/1K)": (_TRAIN_TIME_COLUMN, False, lambda v: f"{v:.2f}"), | |
| "Infer (s/1K)": (_PREDICT_TIME_COLUMN, False, lambda v: f"{v:.3f}"), | |
| } | |
| _OVERVIEW_COLUMNS: list[tuple[str, str, Subset, str | None]] = [ | |
| ("Overall", "overall", Subset(tasks="all", datasets="all"), None), | |
| ("Fit (s/1K)", "cost", Subset(tasks="all", datasets="all"), _TRAIN_TIME_COLUMN), | |
| ("Infer (s/1K)", "cost", Subset(tasks="all", datasets="all"), _PREDICT_TIME_COLUMN), | |
| ("Class.", "task", Subset(tasks="classification", datasets="all"), None), | |
| ("Regr.", "task", Subset(tasks="regression", datasets="all"), None), | |
| ("Binary", "task", Subset(tasks="binary", datasets="all"), None), | |
| ("Multi.", "task", Subset(tasks="multiclass", datasets="all"), None), | |
| ("Small", "size", Subset(tasks="all", datasets="small"), None), | |
| ("Medium", "size", Subset(tasks="all", datasets="medium"), None), | |
| ("Balanced", "target", Subset(tasks="all", datasets="balanced"), None), | |
| ("Imbalanced", "target", Subset(tasks="all", datasets="imbalanced"), None), | |
| ("Extreme", "target", Subset(tasks="all", datasets="extreme"), None), | |
| ] | |
| _GROUP_TITLE = {"overall": "Overall", "cost": "What it costs", "task": "By Task", "size": "By Dataset Size", "target": "By Target"} | |
| #: The overview's column groups that start folded into a stub column (main.taOverviewTable opens them). | |
| _FOLDING_GROUPS = ("task", "size", "target") | |
| #: How the Method header's key names a system's tag (constants.tag_emoji), which follows its name. | |
| _SHADE_KEY = {"closed-source-api": "API system", "with-llm": "with LLMs"} | |
| # Hover tooltips for the overview's subset column headers (rendered as a native `title=`). | |
| # Size definitions are reused from DATASET_SIZE_NOTE so they can't drift from the subset blurbs. | |
| _COLUMN_TOOLTIPS: dict[str, str] = { | |
| "Overall": "All tasks across every dataset size. This is the headline ranking.", | |
| "Fit (s/1K)": ( | |
| "Median seconds to train per 1000 rows, across all datasets, for the variant shown. " | |
| "Tuning fits 200 configurations, so a tuned variant costs far more than its default. " | |
| "Lower is better." | |
| ), | |
| "Infer (s/1K)": ( | |
| "Median seconds to predict per 1000 rows, across all datasets, for the variant shown. " | |
| "Lower is better." | |
| ), | |
| "Class.": "Classification tasks only (binary + multiclass).", | |
| "Regr.": "Regression tasks only.", | |
| "Binary": "Binary classification tasks only.", | |
| "Multi.": "Multiclass classification tasks only.", | |
| "Small": DATASET_SIZE_NOTE["small"], | |
| "Medium": DATASET_SIZE_NOTE["medium"], | |
| "Balanced": DATASET_SIZE_NOTE["balanced"], | |
| "Imbalanced": DATASET_SIZE_NOTE["imbalanced"], | |
| "Extreme": DATASET_SIZE_NOTE["extreme"], | |
| } | |
| #: The top three of each overview column get a medal, in a slot every number cell reserves so the | |
| #: digits line up whether or not one is present (`.ta-medal` in main.py). The full leaderboard table | |
| #: has no medals: its # column already says the same thing. | |
| _MEDALS = {1: "🥇", 2: "🥈", 3: "🥉"} | |
| #: The organization logos, served as static files (main.py registers the folder). | |
| LOGO_DIR = Path(__file__).parent / "assets" / "logos" | |
| #: Organization (the `Organization` column, from the upstream `MethodMetadata.organization`) -> | |
| #: its logo in `LOGO_DIR`. An organization without one shows its initial instead. | |
| ORGANIZATION_LOGOS = { | |
| "Amazon": "amazon.png", | |
| "fast.ai": "fastai.png", | |
| "Google Research": "google.png", | |
| "Inria": "soda-inria.png", | |
| "Layer 6 AI": "layer6.png", | |
| "Lexsi Labs": "lexsi-labs.png", | |
| "LG AI Research": "lg.svg", | |
| "Microsoft": "microsoft.png", | |
| "Nanjing University": "lamda.png", | |
| "Nums AI": "nums-ai.png", | |
| "NVIDIA": "nvidia.png", | |
| "Perpetual ML": "perpetual-ml.png", | |
| "Prior Labs": "priorlabs.png", | |
| "SAP": "sap.png", | |
| "scikit-learn": "scikit-learn.png", | |
| "Stable AI & Tsinghua University": "limix.png", | |
| "Stanford University": "stanford.png", | |
| "Synthefy": "synthefy.png", | |
| "Technion": "technion.svg", | |
| "UC San Diego": "ucsd.png", | |
| "University of Freiburg": "freiburg.png", | |
| "University of Stuttgart & Inria": "inria.png", | |
| "University of Technology Nuremberg": "utn.png", | |
| "University of Washington": "uw.svg", | |
| "Xiaomi": "xiaomi.png", | |
| "Yandex": "yandex.png", | |
| "Yandex Research": "yandex.png", | |
| "Yhat Labs": "yhat-labs.svg", | |
| } | |
| #: Methods that show their own project's logo rather than their organization's, by name prefix. | |
| METHOD_LOGOS = {"AutoGluon": "autogluon.png"} | |
| def org_logo(organization: str, method: str = "") -> str: | |
| """A method's organization as a small tile: its logo, its initial without one, blank when unknown. | |
| A method named in `METHOD_LOGOS` shows that logo instead, with the organization in its hover. | |
| """ | |
| project = next((p for p in METHOD_LOGOS if method.startswith(p)), None) | |
| if project is not None: | |
| name = html.escape(f"{project} ({organization})" if organization else project, quote=True) | |
| logo = METHOD_LOGOS[project] | |
| elif not organization: | |
| return '<span class="ta-org ta-org-none" aria-hidden="true"></span>' | |
| else: | |
| name = html.escape(organization, quote=True) | |
| logo = ORGANIZATION_LOGOS.get(organization) | |
| if logo is None: | |
| return f'<span class="ta-org ta-org-x" title="{name}">{name[:1].upper()}</span>' | |
| # Relative to the working directory, which is how Gradio resolves a static file's path. | |
| src = (Path(os.path.relpath(LOGO_DIR)) / logo).as_posix() | |
| return ( | |
| f'<span class="ta-org" title="{name}">' | |
| f'<img src="/gradio_api/file={src}" alt="{name}" width="20" height="20"></span>' | |
| ) | |
| # The Summary's two metrics, the ones the figures show too: key -> (csv column, higher_is_better, | |
| # formatter). The headline group shows both; a breakdown one at a time, Elo first. | |
| _OVERVIEW_METRIC_SPECS: dict[str, tuple] = { | |
| "elo": ("Elo [⬆️]", True, lambda v: str(int(round(v)))), | |
| "imp": ("Improvability (%) [⬇️]", False, lambda v: f"{v:.1f}"), | |
| } | |
| # What each metric means, in its headline column's hover text. | |
| OVERVIEW_METRIC_TLDR = { | |
| "elo": "Pairwise win-rate rating (a 400-point gap ≈ a 91% win rate). Higher is better.", | |
| "imp": "How much lower the best model's error is than this one's, per dataset. Lower is better.", | |
| } | |
| #: Selector value -> the overview column it points at. A selection of "all" points at nothing, | |
| #: because the Overall column already is that view. | |
| _TASK_TO_COLUMN = {"classification": "Class.", "regression": "Regr.", "binary": "Binary", "multiclass": "Multi."} | |
| _DATASET_TO_COLUMN = { | |
| "small": "Small", | |
| "medium": "Medium", | |
| "balanced": "Balanced", | |
| "imbalanced": "Imbalanced", | |
| "extreme": "Extreme", | |
| } | |
| #: "I care about" -> the cost columns it points at; end-to-end time is the fit and the prediction together. | |
| _CARE_TO_COLUMNS = { | |
| "train": ("Fit (s/1K)",), | |
| "infer": ("Infer (s/1K)",), | |
| "total": ("Fit (s/1K)", "Infer (s/1K)"), | |
| } | |
| def _selected_overview_columns(tasks: str, datasets: str, care: str) -> list[str]: | |
| """The overview columns matching the current selectors, in the order they should lead.""" | |
| wanted = [*_CARE_TO_COLUMNS.get(care, ()), _TASK_TO_COLUMN.get(tasks), _DATASET_TO_COLUMN.get(datasets)] | |
| return [c for c in wanted if c] | |
| _SUBSET_FIELDS = [ | |
| "base", "TypeName", "shade", "variant", "url", "verified", "imputed", "commercial", "license", | |
| "fits_at_predict", "organization", "ci", "leakage", "leakage_datasets", | |
| ] | |
| def _subset_rows(df: pd.DataFrame, column: str) -> pd.DataFrame: | |
| """A subset's rows, `Model` split into base/variant/url and `column` renamed to `val`. | |
| Systems stay in. Which entrants compete is decided by the pool the reader selected, and | |
| the overview reports the numbers computed for that pool, so dropping a competitor here | |
| would show a field the numbers were not computed against. | |
| """ | |
| df = df.dropna(subset=[column]).copy() | |
| parsed = df["Model"].map(parse_model) | |
| df["base"] = [p[0] for p in parsed] | |
| df["variant"] = [p[1] for p in parsed] | |
| df["url"] = [p[2] for p in parsed] | |
| df["imputed"] = df["Imputed"].astype(bool) if "Imputed" in df.columns else False | |
| df["verified"] = df["Verified"] if "Verified" in df.columns else "" | |
| df["commercial"] = commercial_flags(df) | |
| df["license"] = license_names(df) | |
| df["fits_at_predict"] = fits_at_predict_flags(df) | |
| # Artifacts generated before the column existed have no organizations. | |
| df["organization"] = df["Organization"].fillna("").astype(str) if "Organization" in df.columns else "" | |
| df["ci"] = df[_CI_COLUMN].fillna("").astype(str) if _CI_COLUMN in df.columns else "" | |
| df["leakage"] = leakage_evidence(df) | |
| df["leakage_datasets"] = leakage_datasets(df) | |
| tags = df["Tags"] if "Tags" in df.columns else pd.Series(None, index=df.index) | |
| df["shade"] = [system_shade(t, v) for t, v in zip(df["TypeName"], tags)] | |
| return df.rename(columns={column: "val"})[[*_SUBSET_FIELDS, "val"]] | |
| def _overview_th( | |
| text: str, *, tip: str = "", selected: bool = False, attrs: str = "", key: str = "", classes: Sequence[str] = () | |
| ) -> str: | |
| """A column header `<th>` named ``text``, with ``tip`` as its hover text and a 'help' affordance. | |
| ``attrs`` are further attributes, such as the overview's sort data, and ``classes`` further | |
| classes. ``key`` (HTML) says under the name what the marks in the column mean. | |
| """ | |
| attrs = f" {attrs}" if attrs else "" | |
| classes = [*classes, *(["ta-th-info"] if tip else []), *(["ta-col-sel"] if selected else [])] | |
| if classes: | |
| attrs += f' class="{" ".join(classes)}"' | |
| if tip and "data-col" in attrs: | |
| tip += " Click to sort by it." | |
| if tip: | |
| attrs += f' title="{html.escape(tip, quote=True)}"' | |
| # The name has a span of its own, so the sort arrow (main.py) follows it rather than the key. | |
| return f'<th{attrs}><span class="ta-th-name">{text}</span>{_th_key(key)}</th>' | |
| def _th_key(key: str) -> str: | |
| """What the marks in a column mean, as a muted line under its header's name (HTML). | |
| The break is explicit because the headers do not wrap. | |
| """ | |
| return f'<br><span class="ta-th-key">{key}</span>' if key else "" | |
| #: Shown after the name of a method whose license does not permit commercial use. | |
| NONCOMMERCIAL_MARK = "$" | |
| def noncommercial_mark(license_name: str = "") -> str: | |
| """The marker and its hover text, naming the license when one was declared.""" | |
| hint = "License does not permit commercial use" | |
| if license_name: | |
| hint += f": {license_name}" | |
| return f' <span class="ta-nc" title="{html.escape(hint, quote=True)}">{NONCOMMERCIAL_MARK}</span>' | |
| #: The leakage warning's sign: a red triangle with an exclamation mark, drawn the same in both themes. | |
| LEAKAGE_ICON = ( | |
| '<svg class="ta-lk-tri" viewBox="0 0 16 16" aria-hidden="true"><path d="M8 1.6 15 14.2H1z" ' | |
| 'stroke-width="1.6" stroke-linejoin="round"/><rect x="7.05" y="5.4" width="1.9" height="4.9" rx=".95"/>' | |
| '<circle cx="8" cy="12.1" r="1.05"/></svg>' | |
| ) | |
| #: In the medal slot of a flagged method's quality numbers. | |
| _LEAKAGE_NUMBER = ( | |
| f'<span class="ta-lk-n" title="{html.escape(website_texts.LEAKAGE_NUMBER_HINT, quote=True)}">!</span>' | |
| ) | |
| def leakage_badge(name: str, warning: tuple[str, Sequence[str]]) -> str: | |
| """The red badge after a flagged method's name, for ``warning`` as (evidence link, proven | |
| datasets); its hover card (main.taLeakageCard) gives the evidence, the datasets and a way to | |
| hide the flagged methods.""" | |
| url, datasets = warning | |
| proven = website_texts.LEAKAGE_PROVEN[min(len(datasets), 2)].format(n=len(datasets)) | |
| text = website_texts.LEAKAGE_METHOD_TEXT.format(proven=proven) | |
| attrs = { | |
| "data-lk": "method", | |
| "data-lk-head": f"{website_texts.LEAKAGE_METHOD_HEAD}: {name}", | |
| "data-lk-text": text, | |
| "data-lk-more": website_texts.LEAKAGE_DATASETS.format(", ".join(datasets)) if datasets else "", | |
| "data-lk-url": url, | |
| "data-lk-link": website_texts.LEAKAGE_EVIDENCE, | |
| "data-lk-hide": website_texts.LEAKAGE_HIDE_ONE, | |
| } | |
| data = " ".join(f'{key}="{html.escape(value, quote=True)}"' for key, value in attrs.items()) | |
| label = html.escape(attrs["data-lk-head"], quote=True) | |
| return ( | |
| f'<button type="button" class="ta-lk-badge" {data} aria-label="{label}">' | |
| f"{LEAKAGE_ICON}{html.escape(website_texts.LEAKAGE_BADGE)}</button>" | |
| ) | |
| def _leakage_key(rows_meta: Iterable[dict]) -> str: | |
| """The Method header's key for the leakage warning; its hover card speaks for the whole table.""" | |
| n = sum(1 for m in rows_meta if m["leakage"]) | |
| head = website_texts.LEAKAGE_PAGE_HEAD[1 if n == 1 else 2].format(n=n) | |
| attrs = { | |
| "data-lk": "page", | |
| "data-lk-head": head, | |
| "data-lk-text": website_texts.LEAKAGE_PAGE_TEXT if n == 1 else website_texts.LEAKAGE_PAGE_TEXT_MANY, | |
| "data-lk-hide": website_texts.LEAKAGE_HIDE_ONE if n == 1 else website_texts.LEAKAGE_HIDE, | |
| } | |
| data = " ".join(f'{key}="{html.escape(value, quote=True)}"' for key, value in attrs.items()) | |
| return ( | |
| f'<button type="button" class="ta-lk-key" {data} aria-label="{html.escape(head, quote=True)}">' | |
| f"{LEAKAGE_ICON} {html.escape(website_texts.LEAKAGE_KEY)}</button>" | |
| ) | |
| def _shade_scale(value: float, *, log_scale: bool) -> float | None: | |
| """The value in the space the heatmap shades in, or ``None`` when it cannot be placed. | |
| The cost columns are shaded in log space, as the full leaderboard table already does | |
| (`_LB_LOG_SCALED`): runtimes span four orders of magnitude, and a linear ramp painted every | |
| model the same green with only the slowest one red, so a 10x speed difference near the fast | |
| end was invisible. A non-positive runtime has no logarithm and gets the neutral shade. | |
| """ | |
| if not log_scale: | |
| return float(value) | |
| if value <= 0: | |
| return None | |
| return math.log10(value) | |
| _HEAT_STOPS = [(0.0, (28, 120, 62)), (0.5, (138, 122, 36)), (1.0, (160, 58, 58))] | |
| # The same ramp for the opt-in light theme (main.LIGHT_CSS): each stop keeps its hue and is lifted | |
| # to a pastel that dark digits read on. Cells carry it as `--ta-heat-light`, which only the light | |
| # theme reads; the leaderboard table frame carries the same stops upstream. | |
| HEAT_STOPS_LIGHT = [(0.0, (135, 201, 160)), (0.5, (224, 204, 123)), (1.0, (223, 144, 144))] | |
| # The Summary's in-cell bars, whose colour says how good a value is: amber for the best (0) through | |
| # violet to a muted violet for the worst (1), per theme. Hue and lightness change together, so the | |
| # ramp reads without telling red from green, and it stands apart from the heat cells after the | |
| # divider. The dark theme's amber is deep enough to keep the white digits on it at 4.3:1. | |
| BAR_STOPS = [(0.0, (158, 104, 10)), (0.5, (138, 90, 168)), (1.0, (74, 60, 110))] | |
| BAR_STOPS_LIGHT = [(0.0, (242, 194, 90)), (0.5, (199, 163, 221)), (1.0, (221, 211, 236))] | |
| def _interp_color(frac: float, stops: list = _HEAT_STOPS) -> str: | |
| """Map 0 (best) .. 1 (worst) to a green->olive->red hex (by default, readable on dark bg).""" | |
| frac = max(0.0, min(1.0, frac)) | |
| for (f0, c0), (f1, c1) in zip(stops, stops[1:]): | |
| if frac <= f1: | |
| t = 0 if f1 == f0 else (frac - f0) / (f1 - f0) | |
| r, g, b = (round(a + (b_ - a) * t) for a, b_ in zip(c0, c1)) | |
| return f"#{r:02x}{g:02x}{b:02x}" | |
| return "#{:02x}{:02x}{:02x}".format(*stops[-1][1]) | |
| def _heat_style(frac: float) -> str: | |
| """The inline style of a shaded cell: the dark shade, plus the light one for the light theme.""" | |
| return f"background:{_interp_color(frac)};color:#f7f7f7;--ta-heat-light:{_interp_color(frac, HEAT_STOPS_LIGHT)};" | |
| _OVERVIEW_CSS = """ | |
| <style> | |
| .ta-overview { border-collapse: collapse; width: 100%; font-size: 1.08em; } | |
| .ta-overview th, .ta-overview td { padding: 5px 9px; text-align: center; border: 1px solid #ffffff14; } | |
| .ta-overview thead th { background: #1b1b22; font-weight: 600; position: sticky; z-index: 5; box-shadow: inset 0 1px 0 #ffffff14, inset 0 -1px 0 #ffffff14; } | |
| /* Subset column headers carry an explanatory tooltip (title=); hint it with a help cursor + dotted underline. */ | |
| .ta-overview thead th.ta-th-info { cursor: help; text-decoration: underline; text-decoration-style: dotted; text-decoration-color: #ffffff66; text-underline-offset: 3px; } | |
| .ta-overview thead tr:first-child th { top: 0; } | |
| .ta-overview thead tr:nth-child(2) th { top: 33px; } | |
| .ta-overview .ta-group-h { border-bottom: 2px solid #ffffff33; } | |
| .ta-overview td.ta-model-cell { text-align: left; white-space: nowrap; } | |
| /* Pinned to the same grey as the leaderboard table's variant tag (--muted there), | |
| so a "(tuned + ensembled)" reads identically in both tables. */ | |
| .ta-overview .ta-variant { color: #9b9a92; font-weight: 400; font-size: 0.9em; } | |
| /* Numbers match the rest of the table text in size; bold keeps them emphasized. */ | |
| .ta-overview td.ta-num { font-weight: 600; } | |
| .ta-overview td.ta-na { color: #777; } | |
| .ta-overview td.ta-model-cell a { text-decoration: underline; text-decoration-style: dotted; text-underline-offset: 3px; } | |
| .ta-overview td.ta-model-cell a:hover { text-decoration-style: solid; } | |
| .ta-link-icon { font-size: 0.78em; opacity: 0.65; margin-left: 2px; } | |
| .ta-imp { color: #e6c14d; font-weight: 700; margin-left: 1px; } | |
| /* The license marker: the same slot and weight as the imputation star, its own colour. */ | |
| .ta-nc { color: #f0a35a; font-weight: 700; margin-left: 2px; cursor: help; } | |
| .ta-verified { font-size: 0.85em; } | |
| .ta-pill { padding: 1px 7px; border-radius: 999px; font-size: 0.95em; } | |
| .ta-scroll { | |
| overflow: auto; | |
| max-height: 680px; | |
| border: 1px solid #ffffff1f; | |
| border-radius: 10px; | |
| scrollbar-width: thin; | |
| scrollbar-color: #ffffff3a transparent; | |
| scrollbar-gutter: stable; | |
| } | |
| .ta-scroll::-webkit-scrollbar { width: 11px; height: 11px; } | |
| .ta-scroll::-webkit-scrollbar-track { background: transparent; } | |
| .ta-scroll::-webkit-scrollbar-thumb { background: #ffffff33; border-radius: 8px; border: 3px solid transparent; background-clip: content-box; } | |
| .ta-scroll::-webkit-scrollbar-thumb:hover { background: #ffffff5c; background-clip: content-box; } | |
| .ta-scroll::-webkit-scrollbar-corner { background: transparent; } | |
| .ta-cap { font-size: 0.85em; opacity: 0.8; margin: 6px 0 4px 0; } | |
| </style> | |
| """ | |
| # --------------------------------------------------------------------------- # | |
| # The selector card | |
| # --------------------------------------------------------------------------- # | |
| class Choice: | |
| """One chip of a selector row. | |
| Neighbouring chips of one ``section`` share a group, under the caption that key has in | |
| ``SELECTOR_CAPTIONS``. | |
| ``locked`` says why the chip cannot be clicked: one that is always on, or a category nobody has | |
| submitted to yet. ``on`` is a toggle's or switch's state on load. | |
| """ | |
| value: str | |
| label: str | |
| hint: str = "" | |
| section: str = "" | |
| locked: str = "" | |
| on: bool = False | |
| class ChoiceRow: | |
| """One row of the selector card: a caption and the chips that set the selection field ``key``. | |
| ``kind`` is "one" (pick one chip; the first is the default), "many" (toggle any number) or | |
| "switch" (each chip is an on/off setting of its own, named by its value). A ``counted`` row | |
| shows on each chip how many datasets the leaderboard would cover with that chip picked. | |
| """ | |
| key: str | |
| caption: str | |
| hint: str | |
| choices: Sequence[Choice] | |
| kind: str = "one" | |
| counted: bool = False | |
| def _plain(label: str) -> str: | |
| """A chip label as the spec line names it: no leading emoji, and lower-case unless an acronym.""" | |
| text = re.sub(r"^[^\w(]+", "", label) | |
| return text[:1].lower() + text[1:] if text[1:2].islower() else text | |
| def _selector_chip(row: ChoiceRow, choice: Choice, picked: bool) -> str: | |
| if row.kind == "one": | |
| state = f'role="radio" aria-checked="{str(picked).lower()}"' | |
| elif row.kind == "many": | |
| state = f'aria-pressed="{str(choice.on).lower()}"' | |
| else: | |
| state = f'role="switch" aria-checked="{str(choice.on).lower()}"' | |
| if choice.locked: | |
| state += ' aria-disabled="true" data-locked' | |
| mark = "" if row.kind == "one" else '<span class="ta-chip-mark" aria-hidden="true"></span>' | |
| count = '<span class="ta-chip-n"></span>' if row.counted else "" | |
| return ( | |
| f'<button type="button" class="ta-chip" {state} data-value="{choice.value}" ' | |
| f'data-hint="{html.escape(choice.hint)}" title="{html.escape(choice.locked or choice.hint)}">{mark}' | |
| f'<span class="ta-chip-lbl">{html.escape(choice.label)}</span>{count}</button>' | |
| ) | |
| def _selector_row(row: ChoiceRow, inline: bool = False) -> str: | |
| # Runs of neighbouring chips of one section; a run with a section is a group. | |
| runs: list[tuple[str, list[str]]] = [] | |
| for i, choice in enumerate(row.choices): | |
| if not runs or choice.section != runs[-1][0]: | |
| runs.append((choice.section, [])) | |
| runs[-1][1].append(_selector_chip(row, choice, picked=i == 0)) | |
| chips = [] | |
| for section, run in runs: | |
| if section: | |
| caption = ( | |
| f'<span class="ta-sel-sec" title="{html.escape(AXIS_NOTES.get(section, ""))}">' | |
| f"{html.escape(website_texts.SELECTOR_CAPTIONS[section])}</span>" | |
| ) | |
| run = [f'<span class="ta-sel-group">{caption}{"".join(run)}</span>'] | |
| chips += run | |
| group = ' role="radiogroup"' if row.kind == "one" else ' role="group"' | |
| return ( | |
| f'<div class="ta-sel-row{" ta-sel-inline" if inline else ""}" data-row="{row.key}" data-kind="{row.kind}"' | |
| f"{' data-counted' if row.counted else ''}>" | |
| f'<span class="ta-sel-cap" title="{html.escape(row.hint)}">{html.escape(row.caption)}</span>' | |
| f'<div class="ta-sel-chips"{group} aria-label="{html.escape(row.caption)}">{"".join(chips)}</div></div>' | |
| ) | |
| #: The selector bar's icon: three sliders, drawn like the per-dataset bar's list (_PD_ICON). | |
| _SELECTOR_ICON = ( | |
| '<svg viewBox="0 0 20 20" width="18" height="18" fill="none" stroke="currentColor" stroke-width="1.9" ' | |
| 'stroke-linecap="round"><path d="M3 5h14M3 10h14M3 15h14"/>' | |
| '<g fill="currentColor" stroke="none"><circle cx="7" cy="5" r="2.3"/><circle cx="13" cy="10" r="2.3"/>' | |
| '<circle cx="9" cy="15" r="2.3"/></g></svg>' | |
| ) | |
| def _selector_examples(rows: dict[str, ChoiceRow], examples: Sequence[tuple[str, str, str, object]]) -> str: | |
| """The card bar's shortcuts, as (label, hover text, row, value), each a button for taSelector. | |
| A shortcut names a row of the card and the chip it picks, or in a row of toggles a tuple of the | |
| chips it turns on together, or a switch by its value and the state it puts it in (True or False). | |
| Locked chips, such as a system category nobody has submitted to, are left out, and so is a | |
| shortcut with none left; one that names no chip of the card is a mistake in the texts. | |
| """ | |
| switches = {choice.value: choice for row in rows.values() if row.kind == "switch" for choice in row.choices} | |
| buttons = [] | |
| for label, hint, field, value in examples: | |
| if field in rows and rows[field].kind != "switch": | |
| wanted = value if isinstance(value, tuple) and rows[field].kind == "many" else (value,) | |
| chips = {c.value: c for c in rows[field].choices} | |
| choices = [chips.get(v) for v in wanted] | |
| state = " ".join(c.value for c in choices if c and not c.locked) | |
| elif field in switches and isinstance(value, bool): | |
| choices, state = [switches[field]], str(value).lower() if not switches[field].locked else "" | |
| else: | |
| choices, state = [None], "" | |
| if None in choices: | |
| raise ValueError(f"The selector shortcut {label!r} names no chip of the card: {field}={value!r}.") | |
| if not state: | |
| continue | |
| buttons.append( | |
| f'<button type="button" class="ta-sel-ex" data-field="{field}" data-value="{html.escape(state)}" ' | |
| f'aria-pressed="false" title="{html.escape(hint)}">{html.escape(label)}</button>' | |
| ) | |
| return "".join(buttons) | |
| def selector_card( | |
| page: str, | |
| lines: Sequence[Sequence[ChoiceRow]], | |
| *, | |
| title: str, | |
| lead: str, | |
| counts: dict[str, int], | |
| count_axes: Sequence[str], | |
| anchor: str, | |
| warnings: Sequence[tuple[str, str, str, str]] = (), | |
| examples: Sequence[tuple[str, str, str, object]] = (), | |
| ) -> gr.HTML: | |
| """Every selector of a page, as one card of chips that answers a click at once. | |
| The card starts closed, as one row like the per-dataset browser's bar: its ``title``, which is the | |
| button that opens it, the ``examples`` (see :func:`_selector_examples`), shortcuts that pick a | |
| chip and open the card on it, and at its end how many datasets the selection covers, open or | |
| closed. Open, the ``lead`` heads the ``lines`` under the bar. Each line holds one row, or several | |
| that read as one choice, the later ones with their caption inline. A click updates the card in the browser | |
| (main.taSelector): the chip, the dataset counts and the ``warnings``, as (selection | |
| field, value, headline, details in Markdown), each shown while that value is selected. The whole | |
| selection then goes to Python as one ``pick`` event, whose listener writes the page's states. | |
| ``counts`` maps the values of ``count_axes`` joined by "|" to the number of datasets that | |
| leaderboard covers; a chip whose combination is missing cannot be picked. | |
| """ | |
| body = "".join( | |
| f'<div class="ta-sel-line">{"".join(_selector_row(row, inline=i > 0) for i, row in enumerate(line))}</div>' | |
| for line in lines | |
| ) | |
| rows = {row.key: row for line in lines for row in line} | |
| shortcuts = _selector_examples(rows, examples) | |
| if shortcuts: | |
| shortcuts = ( | |
| f'<span class="ta-sel-try" role="group" aria-label="{html.escape(website_texts.SELECTOR_TRY)}">' | |
| f"{shortcuts}</span>" | |
| ) | |
| # The whole bar opens and closes the card (main.taSelector); the title is the button a keyboard | |
| # reaches, its chevron the one that shows it opens. | |
| bar = ( | |
| '<div class="ta-figbar ta-pdbar ta-sel-bar">' | |
| f'<span class="ta-pdicon" aria-hidden="true">{_SELECTOR_ICON}</span>' | |
| '<h2 class="ta-figbar-title ta-sel-title"><button type="button" class="ta-sel-toggle" aria-expanded="false">' | |
| f'{html.escape(title)}<span class="ta-sel-chev" aria-hidden="true"></span></button></h2>' | |
| f"{shortcuts}</div>" | |
| ) | |
| warns = "".join( | |
| f'<details class="ta-sel-warn" data-field="{field}" data-when="{value}" hidden>' | |
| f'<summary><span aria-hidden="true">⚠️</span><span class="ta-sel-warn-head">{html.escape(head)}</span>' | |
| '<span class="ta-sel-why"></span></summary>' | |
| f'<div class="ta-sel-warn-body">{render_markdown(body)}</div></details>' | |
| for field, value, head, body in warnings | |
| ) | |
| data = { | |
| "counts": counts, | |
| "axes": list(count_axes), | |
| "none": website_texts.SELECTOR_NO_DATASETS, | |
| "total": website_texts.SELECTOR_TOTAL, | |
| # Every dataset of the page: each row on its first chip. | |
| "whole": counts.get("|".join(rows[axis].choices[0].value for axis in count_axes)), | |
| } | |
| return gr.HTML( | |
| f'<div class="ta-sel" data-page="{page}" data-open="false" data-config="{html.escape(json.dumps(data))}">' | |
| f'{bar}<div class="ta-sel-main"><p class="ta-sel-lead">{html.escape(lead)}</p>{body}</div>' | |
| '<div class="ta-sel-result" aria-live="polite">' | |
| '<p class="ta-sel-total"><b class="ta-sel-num"></b><span class="ta-sel-unit"></span></p></div>' | |
| f'<div class="ta-sel-warns">{warns}</div></div>', | |
| js_on_load="taSelector(element, (selection) => trigger('pick', selection));", | |
| apply_default_css=False, | |
| elem_id=anchor, | |
| elem_classes="ta-sel-block", | |
| ) | |
| #: The view bar's anchor per page. | |
| VIEW_ANCHORS = {"tabarena": "ta-views-anchor", "beyondarena": "beyond-views-anchor"} | |
| def view_bar(slug: str, *, follow: str = "", hint: str = "", extra: str = "") -> str: | |
| """A page's view tabs (``website_texts.VIEW_TABS``), the top of its controls, with ``follow`` (the | |
| Follow picker, HTML) at the right. | |
| Plain buttons: the browser shows the view a tab names (``main.taShowView``), every view being on | |
| the page from the start, so a switch costs no round trip, and marks the page's shown one | |
| whenever the bar is drawn again (``main.taSyncViewTabs``). A tab's hover text says what its view | |
| shows, ending with ``hint``, what the selection covers; ``extra`` (HTML) follows the tabs, such as | |
| a warning, and so does a note that only one view has, shown with it. | |
| """ | |
| hints = dict(website_texts.VIEW_HINTS) | |
| notes = {} | |
| if slug == "beyondarena": | |
| hints.update(website_texts.BEYOND_VIEW_HINTS) | |
| # Its Summary charts the subsets, whose meaning is a Learn more topic. | |
| notes["summary"] = ( | |
| f'<button type="button" class="ta-viewnote-link" onclick="{open_doc("beyondarena", "subsets")}">' | |
| f"{html.escape(website_texts.BEYOND_SUBSETS_LINK)}</button>" | |
| ) | |
| buttons = "".join( | |
| f'<button type="button" class="ta-viewtab" role="tab" data-view="{key}" ' | |
| f'aria-selected="{str(i == 0).lower()}" ' | |
| f'title="{html.escape(" ".join(part for part in (hints.get(key, ""), hint) if part), quote=True)}">' | |
| f"{html.escape(label)}</button>" | |
| for i, (key, label) in enumerate(website_texts.VIEW_TABS) | |
| ) | |
| shown = "".join(f'<span class="ta-viewnote" data-view="{key}">{note}</span>' for key, note in notes.items()) | |
| return ( | |
| f'<div class="ta-viewbar" id="{VIEW_ANCHORS[slug]}" data-page="{slug}">' | |
| f'<nav class="ta-viewtabs" role="tablist" aria-label="Views">{buttons}</nav>' | |
| f'<p class="ta-viewnotes">{shown}{extra}</p>{follow}</div>' | |
| ) | |
| def picked(value: object, labels: dict[str, str]) -> str: | |
| """``value`` when it is one of the ``labels`` keys, else the first key (the row's default). | |
| The selection comes from the browser, so nothing in it is used before it is checked. | |
| """ | |
| return value if isinstance(value, str) and value in labels else next(iter(labels)) | |
| #: Anchor on TabArena's selector card. | |
| CONTROLS_ANCHOR = "ta-controls-anchor" | |
| def make_cross_subset_overview( | |
| data_root: Path, | |
| entrants: str = "models", | |
| tasks: str = "all", | |
| datasets: str = "all", | |
| care: str = "quality", | |
| include_noncommercial: bool = True, | |
| ) -> gr.HTML: | |
| """The Summary: every entrant (rows) on every subset (columns), within one pool, as one table. | |
| A row is one model *variant* — default, tuned, or tuned + ensembled — so its quality and its | |
| cost are always the same run's. | |
| The headline columns, Elo and improvability over all datasets, and the cost columns carry an | |
| in-cell bar as long as the value is high in its column, coloured from amber for the best | |
| values to violet for the worst (BAR_STOPS); a divider follows each of the two groups. The breakdowns by task, size and | |
| target are heat cells, folded into stub columns until opened, and carry both | |
| metrics, of which a switch in each one's heading shows one (main.taOverviewTable). | |
| `entrants` selects the pool. Every column is read from that pool's artifacts, so the whole | |
| grid is one consistent field of competitors. | |
| `tasks`, `datasets` and `care` do not change what the table contains, only where the eye | |
| lands: the column each of them points at leads its group and is tinted, and its breakdown | |
| starts open, so the reader can find their own view without hunting along the header. | |
| Reordering happens inside a group rather than across the whole table, which would leave the | |
| grouped header describing columns that had moved out from under it. | |
| `include_noncommercial` off drops the rows whose license forbids commercial use. A row | |
| filter, unlike the entrant pool: the remaining rows keep their published numbers, which | |
| were computed against the full field. A method with a leakage warning (its `Leakage` column) | |
| carries a badge whose hover gives the evidence, a mark in the medal slot of each quality number, | |
| and no medal: each medal goes to the best method without a warning. Leaving those methods out | |
| is a pool of its own (`data_loading.leakage_pool`), recomputed without them. | |
| """ | |
| # A column is (id, label, group, metric): a cost column reports its own number and has no | |
| # metric; every other one comes twice, once per metric, its id the label and the metric. | |
| columns: list[tuple[str, str, str, str | None]] = [] | |
| vals: dict[str, dict[tuple[str, str], float]] = {} | |
| imps: dict[str, dict[tuple[str, str], bool]] = {} | |
| cis: dict[str, dict[tuple[str, str], str]] = {} | |
| meta: dict[tuple[str, str], dict] = {} | |
| for label, group, subset, value_column in _OVERVIEW_COLUMNS: | |
| path = Path(data_root) / replace(subset, entrants=entrants).rel_path / "website_leaderboard.csv" | |
| if not path.exists(): | |
| continue | |
| df = load_leaderboard_csv(str(path.resolve())) | |
| specs = ( | |
| [(label, None, value_column)] | |
| if value_column is not None | |
| else [(f"{label}|{m}", m, spec[0]) for m, spec in _OVERVIEW_METRIC_SPECS.items()] | |
| ) | |
| for cid, metric, csv_column in specs: | |
| if csv_column not in df.columns: | |
| continue | |
| rows = _subset_rows(df, csv_column) | |
| if not include_noncommercial: | |
| rows = rows[rows["commercial"]] | |
| if rows.empty: | |
| continue | |
| keys = list(zip(rows["base"], rows["variant"])) | |
| vals[cid] = dict(zip(keys, rows["val"])) | |
| imps[cid] = dict(zip(keys, rows["imputed"])) | |
| # The Elo columns carry their bootstrap interval. | |
| if metric == "elo": | |
| cis[cid] = dict(zip(keys, rows["ci"])) | |
| columns.append((cid, label, group, metric)) | |
| for key, (_, row) in zip(keys, rows.iterrows()): | |
| meta.setdefault( | |
| key, | |
| { | |
| "type_name": row["TypeName"], | |
| "shade": row["shade"], | |
| "variant": row["variant"], | |
| "url": row["url"], | |
| "verified": row["verified"], | |
| "commercial": bool(row["commercial"]), | |
| "license": row["license"], | |
| "fits_at_predict": bool(row["fits_at_predict"]), | |
| "organization": row["organization"], | |
| "leakage": (row["leakage"], row["leakage_datasets"]) if row["leakage"] else None, | |
| }, | |
| ) | |
| if not columns: | |
| return _no_overview() | |
| # Each selected column leads its own group; a breakdown runs its Elo columns, then the same | |
| # columns in improvability. | |
| highlighted = [c for c in _selected_overview_columns(tasks, datasets, care) if any(c == col[1] for col in columns)] | |
| folding = [g for g in _FOLDING_GROUPS if any(col[2] == g for col in columns)] | |
| ordered = [] | |
| for group in dict.fromkeys(col[2] for col in columns): | |
| items = [col for col in columns if col[2] == group] | |
| items = [col for col in items if col[1] in highlighted] + [col for col in items if col[1] not in highlighted] | |
| if group in folding: | |
| items = [col for m in _OVERVIEW_METRIC_SPECS for col in items if col[3] == m] | |
| ordered += items | |
| columns = ordered | |
| def higher(col: tuple) -> bool: | |
| return _FIXED_COLUMN_SPECS[col[1]][1] if col[3] is None else _OVERVIEW_METRIC_SPECS[col[3]][1] | |
| def fmt(col: tuple): | |
| return _FIXED_COLUMN_SPECS[col[1]][2] if col[3] is None else _OVERVIEW_METRIC_SPECS[col[3]][2] | |
| lead = next((col for col in columns if col[2] == "overall" and col[3] == "elo"), columns[0]) | |
| worst = float("-inf") if higher(lead) else float("inf") | |
| entries = sorted(meta, key=lambda k: vals[lead[0]].get(k, worst), reverse=higher(lead)) | |
| rank_by_col, medal_by_col, bounds = {}, {}, {} | |
| for col in columns: | |
| values = vals[col[0]] | |
| ranked = sorted(values, key=lambda b: values[b], reverse=higher(col)) | |
| rank_by_col[col[0]] = {b: i + 1 for i, b in enumerate(ranked)} | |
| # A method with a leakage warning wins no medal; the next one without a warning does. | |
| clean = [b for b in ranked if not meta[b]["leakage"]] | |
| medal_by_col[col[0]] = {b: _MEDALS[i + 1] for i, b in enumerate(clean[:3])} | |
| shaded = [_shade_scale(v, log_scale=col[3] is None) for v in values.values()] | |
| shaded = [v for v in shaded if v is not None] | |
| bounds[col[0]] = (min(shaded), max(shaded)) if shaded else (0.0, 1.0) | |
| # A divider closes each group before the breakdowns: the headline metrics, then the costs. | |
| before = [col for col in columns if col[2] not in folding] | |
| ends = {[c for c in before if c[2] == g][-1][0]: g for g in dict.fromkeys(c[2] for c in before)} | |
| auto_open = " ".join(dict.fromkeys(col[2] for col in columns if col[1] in highlighted and col[2] in folding)) | |
| def data_attrs(col: tuple) -> str: | |
| return f' data-g="{col[2]}" data-m="{col[3]}"' if col[2] in folding else "" | |
| # -- Grouped header (two rows); the model column stays in view when the table scrolls sideways, | |
| # the rank column sliding under it. A row's family shows in the colour of its name, keyed in | |
| # the Method header, so there is no type column. | |
| # Every number column sorts on a click (main.taOverviewTable), best first; the headline Elo | |
| # is the order the rows arrive in. | |
| rows_meta = meta.values() | |
| marks = " ".join( | |
| f'<span class="ta-th-mark">{mark}</span>' | |
| for mark, shown in ( | |
| ('<span class="ta-verified">✔️</span> verified', any(m["verified"] == "✔️" for m in rows_meta)), | |
| (f'<span class="ta-nc">{NONCOMMERCIAL_MARK}</span> non-commercial', any(not m["commercial"] for m in rows_meta)), | |
| ('<span class="ta-link-icon">↗</span> paper or code', any(m["url"] for m in rows_meta)), | |
| (_leakage_key(rows_meta), any(m["leakage"] for m in rows_meta)), | |
| *( | |
| (f'<span class="ta-xo-tag">{tag_emoji[shade]}</span> {html.escape(_SHADE_KEY[shade])}', | |
| any(m["shade"] == shade for m in rows_meta)) | |
| for shade in _SHADE_KEY | |
| ), | |
| ) | |
| if shown | |
| ) | |
| column_keys = {} | |
| if any(any(col.values()) for col in imps.values()): | |
| column_keys[lead[0]] = '<span class="ta-imp">*</span> imputed' | |
| if any(m["fits_at_predict"] for m in rows_meta): | |
| column_keys["Fit (s/1K)"] = f'<span class="ta-imp">†</span> {_FIT_AT_PREDICT_KEY}' | |
| row1 = [ | |
| f'<th rowspan="2" class="ta-xo-pos" title="{html.escape(website_texts.OVERVIEW_POS_HINT, quote=True)}">#</th>', | |
| f'<th rowspan="2" class="ta-xo-model">Method{_overview_find()}{_th_key(_method_legend(rows_meta))}' | |
| f"{_th_key(marks)}</th>", | |
| ] | |
| row2 = [] | |
| for group, items in groupby(columns, key=lambda col: col[2]): | |
| items = list(items) | |
| sep = " ta-xo-sep" if group in ends.values() else "" | |
| if group in folding: | |
| switch = "".join( | |
| f'<button type="button" data-gm="{group}:{m}" aria-pressed="{str(m == "elo").lower()}">' | |
| f"{html.escape(website_texts.OVERVIEW_METRIC_NAMES[m][1])}</button>" | |
| for m in _OVERVIEW_METRIC_SPECS | |
| ) | |
| row1.append( | |
| f'<th colspan="{len(items) // len(_OVERVIEW_METRIC_SPECS)}" class="ta-group-h" data-g="{group}">' | |
| f'{_GROUP_TITLE[group]}<span class="ta-xo-mswitch" role="group">{switch}</span>' | |
| f'<button type="button" class="ta-xo-fold" data-fold="{group}" ' | |
| f'title="{html.escape(website_texts.OVERVIEW_FOLD, quote=True)}">✕</button></th>' | |
| ) | |
| else: | |
| row1.append(f'<th colspan="{len(items)}" class="ta-group-h{sep}">{_GROUP_TITLE[group]}</th>') | |
| for i, col in enumerate(items): | |
| cid, label, _, metric = col | |
| if metric is None: | |
| text, tip = label, _COLUMN_TOOLTIPS.get(label, "") | |
| elif group == "overall": | |
| text = website_texts.OVERVIEW_METRIC_NAMES[metric][0] | |
| tip = f"{OVERVIEW_METRIC_TLDR[metric]} {_COLUMN_TOOLTIPS['Overall']}" | |
| else: | |
| text = label | |
| tip = f"{_COLUMN_TOOLTIPS.get(label, '')} In {website_texts.OVERVIEW_METRIC_NAMES[metric][0].lower()}." | |
| attrs = ( | |
| f'data-col="{columns.index(col)}" data-label="{html.escape(cid, quote=True)}" ' | |
| f'data-dir="{"desc" if higher(col) else "asc"}"' | |
| + (f' aria-sort="{"descending" if higher(col) else "ascending"}"' if col == lead else "") | |
| + data_attrs(col) | |
| ) | |
| row2.append( | |
| _overview_th( | |
| text, | |
| tip=tip, | |
| selected=label in highlighted, | |
| attrs=attrs, | |
| key=column_keys.get(cid, ""), | |
| classes=("ta-xo-sep",) if cid in ends else (), | |
| ) | |
| ) | |
| for group in folding: | |
| n = sum(1 for col in columns if col[2] == group) // len(_OVERVIEW_METRIC_SPECS) | |
| row1.append( | |
| f'<th rowspan="2" class="ta-xo-stub" data-stub="{group}">' | |
| f'<button type="button" class="ta-xo-unfold" data-unfold="{group}">{_GROUP_TITLE[group]}</button>' | |
| f"{_th_key(website_texts.OVERVIEW_STUB.format(n=n))}</th>" | |
| ) | |
| header = f"<thead><tr>{''.join(row1)}</tr><tr>{''.join(row2)}</tr></thead>" | |
| stubs = "".join(f'<td class="ta-xo-stub" data-stub="{group}"></td>' for group in folding) | |
| added = _recent_additions() | |
| # -- Body | |
| body = [] | |
| for key in entries: | |
| m = meta[key] | |
| color = type_color(m["type_name"], m["shade"]) | |
| # The name and what follows it each stay whole; a long one moves its suffix to a second line. | |
| suffix = f'<span class="ta-variant">({html.escape(m["variant"])})</span>' if m["variant"] else "" | |
| if m["shade"] in tag_emoji: | |
| suffix += ( | |
| f' <span class="ta-xo-tag" title="{html.escape(system_shade_label[m["shade"]], quote=True)}">' | |
| f"{tag_emoji[m['shade']]}</span>" | |
| ) | |
| if m["verified"] == "✔️": | |
| suffix += ' <span class="ta-verified" title="Verified implementation">✔️</span>' | |
| if not m["commercial"]: | |
| suffix += noncommercial_mark(m["license"]) | |
| if m["url"]: | |
| suffix += '<span class="ta-link-icon">↗</span>' | |
| name = f'<span class="ta-xo-n">{html.escape(key[0])}</span> <span class="ta-xo-suffix">{suffix}</span>' | |
| if m["url"]: | |
| name = ( | |
| f'<a href="{html.escape(m["url"])}" target="_blank" rel="noopener" title="Open its paper or code" ' | |
| f'style="color:{color}">{name}</a>' | |
| ) | |
| else: | |
| name = f'<span style="color:{color}">{name}</span>' | |
| if key[0] in added: | |
| name += f'<span class="ta-new" title="{html.escape(added[key[0]], quote=True)}">new</span>' | |
| if m["leakage"]: | |
| name += leakage_badge(key[0], m["leakage"]) | |
| cells = [ | |
| f'<td class="ta-xo-pos">{rank_by_col[lead[0]].get(key, "–")}</td>', | |
| ( | |
| f'<td class="ta-xo-model"><span class="ta-xo-id">{org_logo(m["organization"], key[0])}{name}</span>' | |
| # Follows the method in the figures below (main.taToggleFollow); outside the link. | |
| f'<button type="button" class="ta-follow-btn" data-follow="{html.escape(key[0], quote=True)}" ' | |
| f'aria-pressed="false" title="{html.escape(website_texts.FOLLOW_ROW.format(key[0]), quote=True)}">' | |
| "◎</button></td>" | |
| ), | |
| ] | |
| for col in columns: | |
| cid, label, group, metric = col | |
| classes = ["ta-num"] | |
| if cid in ends: | |
| classes.append("ta-xo-sep") | |
| if label in highlighted: | |
| classes.append("ta-col-sel") | |
| val = vals[cid].get(key) | |
| if val is None: | |
| cells.append(f'<td class="ta-na{" ta-xo-sep" if cid in ends else ""}"{data_attrs(col)}>–</td>') | |
| continue | |
| # Leakage inflates quality, not cost: only the quality numbers carry its mark. | |
| if m["leakage"] and metric is not None: | |
| medal = _LEAKAGE_NUMBER | |
| else: | |
| medal = medal_by_col[cid].get(key, "") | |
| lo, hi = bounds[cid] | |
| scaled = _shade_scale(val, log_scale=metric is None) | |
| if hi <= lo or scaled is None: | |
| frac_best = 0.5 | |
| else: | |
| frac_best = (scaled - lo) / (hi - lo) if higher(col) else (hi - scaled) / (hi - lo) | |
| rank = rank_by_col[cid][key] | |
| hover = f"#{rank} of {len(rank_by_col[cid])} · {_COLUMN_NAMES.get(label, label)}" | |
| if metric is not None: | |
| hover += f" · {website_texts.OVERVIEW_METRIC_NAMES[metric][0].lower()}" | |
| ci = cis.get(cid, {}).get(key, "") | |
| if ci: | |
| hover += f" · 95% CI {ci}" | |
| imp = '<sup class="ta-imp" title="Score is (partly) imputed">*</sup>' if imps[cid].get(key) else "" | |
| if label == "Fit (s/1K)" and m["fits_at_predict"]: | |
| imp += f'<sup class="ta-imp" title="{_FIT_AT_PREDICT_HINT}">†</sup>' | |
| if group in folding: | |
| style = _heat_style(1 - frac_best) | |
| else: | |
| # Before the dividers a bar as long as the value sits high in its column (on the log | |
| # scale for the costs), so a lower improvability or a faster time is a shorter bar; | |
| # its colour (BAR_STOPS) says whether that is good. The headline columns read larger, and | |
| # the headline Elo shows its interval. | |
| classes.append("ta-bar") | |
| classes.append("ta-xo-lead" if group == "overall" else "ta-xo-cost") | |
| length = 0.5 if hi <= lo or scaled is None else (scaled - lo) / (hi - lo) | |
| style = ( | |
| f"--bar-dark:{_interp_color(1 - frac_best, BAR_STOPS)};" | |
| f"--bar-light:{_interp_color(1 - frac_best, BAR_STOPS_LIGHT)};" | |
| f"--w:{max(4, round(length * 100))}%;" | |
| ) | |
| ci = f'<span class="ta-ci">{html.escape(ci.replace("-", "−"))}</span>' if ci and col == lead else "" | |
| cells.append( | |
| f'<td class="{" ".join(classes)}"{data_attrs(col)} style="{style}" data-v="{val}" ' | |
| f'data-rank="{rank}" title="{html.escape(hover, quote=True)}">' | |
| f'<span class="ta-cell"><span class="ta-medal">{medal}</span>' | |
| f'<span class="ta-val">{fmt(col)(val)}{imp}</span>{ci}</span></td>' | |
| ) | |
| flagged = ' class="ta-lk-row"' if m["leakage"] else "" | |
| body.append( | |
| f'<tr data-name="{html.escape(key[0].lower(), quote=True)}"{flagged}>{"".join(cells)}{stubs}</tr>' | |
| ) | |
| table = f'<table class="ta-xo" data-auto="{auto_open}">{header}<tbody>{"".join(body)}</tbody></table>' | |
| return gr.HTML( | |
| f'{_OVERVIEW_CSS}<div class="ta-xo-scroll">{table}</div>', | |
| js_on_load="taOverviewTable(element);", | |
| apply_default_css=False, | |
| elem_classes="ta-overview-block", | |
| ) | |
| def _no_overview() -> gr.HTML: | |
| """A note in place of the overview, when its pool has no data.""" | |
| return gr.HTML("<p>No overview data available.</p>", elem_classes="ta-overview-block") | |
| #: An overview column as its hover text names it. | |
| _COLUMN_NAMES = { | |
| "Overall": "all datasets", | |
| "Fit (s/1K)": "train time", | |
| "Infer (s/1K)": "inference time", | |
| "Class.": "classification", | |
| "Regr.": "regression", | |
| "Binary": "binary", | |
| "Multi.": "multiclass", | |
| "Small": "small datasets", | |
| "Medium": "medium datasets", | |
| "Balanced": "balanced targets", | |
| "Imbalanced": "imbalanced targets", | |
| "Extreme": "extreme imbalance", | |
| } | |
| #: How many of the latest releases that added a method the "new" badge covers. | |
| _NEW_RELEASES = 3 | |
| def _family_key(type_name: str, shade: str | None) -> str: | |
| """The legend entry a row belongs to: its system shade, else its family, Other folded into Baseline.""" | |
| return shade or (Constants.baseline if type_name == Constants.other else type_name) | |
| def _recent_additions() -> dict[str, str]: | |
| """The methods the latest releases added, as name -> when (from the version history).""" | |
| releases, _ = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT) | |
| return {name: f"New in {r.version}, {r.when}" for name, r in _added_entrants(releases)[:_NEW_RELEASES]} | |
| #: The families in the order the legend names them, as (family, short name, full name); Baseline and | |
| #: Other share a colour, so one entry. The system shades follow, by _SHADE_SHORT. | |
| _LEGEND_FAMILIES = [ | |
| (Constants.foundational, "Foundation", "Foundation Model"), | |
| (Constants.neural_network, "Neural", "Neural Network"), | |
| (Constants.tree, "Tree", "Tree-based"), | |
| (Constants.baseline, "Baseline", "Baseline / Other"), | |
| (Constants.system, "System", "System"), | |
| ] | |
| _SHADE_SHORT = {"closed-source-api": "API", "with-llm": "LLM"} | |
| def _method_legend(rows_meta: Iterable[dict]) -> str: | |
| """The colours of the names, a line under the Method header (HTML). | |
| Each family or system shade a row on screen carries, by a short name in its colour, the full | |
| name on hover, so the key fits the column and the table needs no legend row. | |
| """ | |
| present = {_family_key(m["type_name"], m["shade"]) for m in rows_meta} | |
| entries = [ | |
| (model_type_color.get(name, "#9e9e9e"), short, full) | |
| for name, short, full in _LEGEND_FAMILIES | |
| if name in present | |
| ] | |
| entries += [ | |
| (color, _SHADE_SHORT[shade], system_shade_label[shade]) | |
| for shade, color in system_shade_color.items() | |
| if shade in present | |
| ] | |
| return " ".join( | |
| f'<span class="ta-th-mark ta-xo-fam" style="color:{c}" title="{html.escape(full, quote=True)}">' | |
| f"{html.escape(short)}</span>" | |
| for c, short, full in entries | |
| ) | |
| #: The Summary's search icon; the box beside it opens when the icon is clicked (main.py). | |
| _SEARCH_ICON = ( | |
| '<svg viewBox="0 0 20 20" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" ' | |
| 'stroke-linecap="round" aria-hidden="true"><circle cx="8.5" cy="8.5" r="5.5"/><path d="m13 13 4 4"/></svg>' | |
| ) | |
| def _overview_find() -> str: | |
| """The Summary's search, in the Method header's corner: an icon opening a box that keeps the rows | |
| whose name holds its text (main.taOverviewTable).""" | |
| find = html.escape(website_texts.OVERVIEW_FIND, quote=True) | |
| return ( | |
| f'<label class="ta-xo-find" title="{find}">{_SEARCH_ICON}' | |
| f'<input type="search" class="ta-xo-search" placeholder="{find}" aria-label="{find}"></label>' | |
| ) | |
| def section_head(title: str, line: str = "", *, anchor: str = "", hint: str = "", actions: str = "") -> str: | |
| """A section's heading: its title, beside it a line saying what the section holds (HTML), and | |
| ``actions`` at the right. ``hint`` is the title's hover text, for what the line leaves out. | |
| """ | |
| anchor = f' id="{anchor}"' if anchor else "" | |
| hint = f' title="{html.escape(hint, quote=True)}"' if hint else "" | |
| line = f'<span class="ta-head-line">{line}</span>' if line else "" | |
| return ( | |
| f'<div class="ta-section-head"{anchor}><div class="ta-section-row">' | |
| f'<div class="ta-head-main"><h2{hint}>{html.escape(title)}</h2>{line}</div>{actions}</div></div>' | |
| ) | |
| #: Where the selected tasks span two kinds, the two the spec line counts the datasets of. | |
| _TASK_HALVES = {"all": ("classification", "regression"), "classification": ("binary", "multiclass")} | |
| def follow_picker(lb: LBContainer) -> str: | |
| """The "Follow" row under the leaderboard's heading: the methods picked here are marked in | |
| every figure and table of the page (``main.taToggleFollow``). | |
| The list holds the subset's methods in leaderboard order, each in its family colour, without | |
| the ones the reader excluded. The picks live on the page rather than in this markup, so a new | |
| selection keeps them; the chips after the button list them and drop one on a click. | |
| """ | |
| try: | |
| df = lb.load_df() | |
| except FileNotFoundError: | |
| return "" | |
| excluded = set(lb.excluded_methods()) | |
| tags = df["Tags"] if "Tags" in df.columns else pd.Series(None, index=df.index) | |
| items, seen = [], set() | |
| for model, type_name, tag in zip(df["Model"], df["TypeName"], tags, strict=True): | |
| base = parse_model(str(model))[0] | |
| if base in seen or base in excluded: | |
| continue | |
| seen.add(base) | |
| items.append( | |
| f'<button type="button" class="ta-follow-item" data-follow="{html.escape(base, quote=True)}" ' | |
| f'aria-pressed="false" style="--fam:{type_color(type_name, system_shade(type_name, tag))}">' | |
| f'<span class="ta-follow-dot"></span>{html.escape(base)}</button>' | |
| ) | |
| find = html.escape(website_texts.FOLLOW_FIND, quote=True) | |
| return ( | |
| '<div class="ta-follow"><details class="ta-follow-pick">' | |
| f'<summary class="ta-viewbtn" title="{html.escape(website_texts.FOLLOW_HINT, quote=True)}">' | |
| f"{website_texts.FOLLOW_LABEL}</summary>" | |
| f'<div class="ta-follow-menu"><input type="search" class="ta-follow-find" placeholder="{find}" ' | |
| f'aria-label="{find}"><div class="ta-follow-list">{"".join(items)}</div></div></details>' | |
| '<span class="ta-follow-chips"></span></div>' | |
| ) | |
| def lb_spec(lb: LBContainer) -> tuple[str, str]: | |
| """What the selected leaderboard covers, as the line beside its section's title, and the notes | |
| behind it as the title's hover text. | |
| The number of datasets leads, split in two by task where the selection spans both kinds; the | |
| other filters and the settings follow, named as the selector card's summary names them. | |
| """ | |
| subset = lb.subset | |
| n = lb.n_datasets | |
| parts = [f"<b>{n} {'dataset' if str(n) == '1' else 'datasets'}</b>"] | |
| halves = _TASK_HALVES.get(subset.tasks) | |
| if halves: | |
| counts = [(LBContainer(lb.data_root, replace(subset, tasks=half), "").n_datasets, half) for half in halves] | |
| parts.append(" / ".join(f"{count} {TASK_LABELS[half].lower()}" for count, half in counts if count)) | |
| else: | |
| parts.append(TASK_LABELS[subset.tasks].lower()) | |
| notes = [] | |
| if isinstance(subset, BeyondSubset): | |
| parts += [BEYOND_SUBSET_LABELS[key] for key in subset.filters if key != subset.tasks] | |
| parts.append(website_texts.BEYOND_SELECTOR_TAIL[0]) | |
| notes += [BEYOND_SUBSET_NOTE[key] for key in subset.filters] | |
| if str(n).isdigit() and int(n) < BEYOND_FEW_DATASETS: | |
| parts.append( | |
| f'<span class="ta-head-warn" title="{html.escape(BEYOND_FEW_DATASETS_NOTE, quote=True)}">' | |
| "⚠️ few datasets</span>" | |
| ) | |
| else: | |
| if subset.datasets != "all": | |
| parts.append(DATASET_LABELS[subset.datasets].lower()) | |
| categories = entrants_categories(subset.entrants) | |
| names = ["models", *(_plain(SYSTEM_CATEGORY_LABELS[key]) for key in categories)] | |
| parts.append(" + ".join(names) if categories else website_texts.SELECTOR_ALONE.format("models")) | |
| notes_on = website_texts.SELECTOR_SETTING_NOTES | |
| parts.append(notes_on["imputed"][0 if subset.imputation == "yes" else 1]) | |
| if subset.splits == "lite": | |
| parts.append(notes_on["lite"][0]) | |
| notes += [entrants_note(subset.entrants), DATASET_SIZE_NOTE.get(subset.datasets, "")] | |
| if not lb.include_noncommercial: | |
| parts.append(website_texts.SELECTOR_SETTING_NOTES["noncommercial"][1]) | |
| if not lb.include_leakage: | |
| parts.append(website_texts.SELECTOR_SETTING_NOTES["leakage"][1]) | |
| return " · ".join(part for part in parts if part), " ".join(note for note in notes if note) | |
| # --------------------------------------------------------------------------- # | |
| # Agentic guide | |
| # --------------------------------------------------------------------------- # | |
| def _hero_toggle(slug: str, kind: str) -> str: | |
| """The hidden checkbox whose state shows one of a page hero's dialogs (see main.taOpenDialog). | |
| Every page is in the document at once, so each page has its own. | |
| """ | |
| return f"ta-{slug}-{kind}-toggle" | |
| def open_doc(slug: str, topic: str) -> str: | |
| """The click handler that opens page ``slug``'s Learn more dialog on ``topic``.""" | |
| return f"taOpenDoc('{_hero_toggle(slug, 'docs')}', '{topic}')" | |
| # A release that adds an entrant opens with "Add new verified model: <name>, …" (the | |
| # update-leaderboard skill writes it that way); the name ends at the first comma, bracket or stop. | |
| _NEW_ENTRANT = re.compile( | |
| r"Add (?:the first )?(?:new )?(?:verified |unverified )?(?:model|system)s?: (.+?)(?:, | \(|\. |;|\.?$)" | |
| ) | |
| def _added_entrants(releases: Sequence[Release]) -> list[tuple[str, Release]]: | |
| """The entrants releases add, newest first, each with the release that added it.""" | |
| added = [] | |
| for release in releases: | |
| first = re.sub(r"\s+", " ", release.source).split("* ", 1)[-1] | |
| match = _NEW_ENTRANT.match(first) | |
| if match: | |
| added.append((match.group(1), release)) | |
| return added | |
| def _newest_entrant(releases: Sequence[Release]) -> str: | |
| """The entrant the newest release that adds one names, or "" when none does.""" | |
| added = _added_entrants(releases) | |
| return added[0][0] if added else "" | |
| def _entrant_facts(df: pd.DataFrame) -> dict[str, str]: | |
| """How many models and systems a leaderboard lists, a method counting once whatever its variants.""" | |
| method = df["Model"].map(lambda m: parse_model(m)[0]) | |
| if "MethodClass" in df.columns: | |
| is_system = df["MethodClass"].eq("system") | |
| else: | |
| is_system = df["TypeName"].eq(Constants.system) | |
| if "Verified" in df.columns: | |
| verified = df["Verified"].astype(str).str.strip().eq("✔️") | |
| else: | |
| verified = pd.Series(False, index=df.index) | |
| n_models = method[~is_system].nunique() | |
| n_systems = method[is_system].nunique() | |
| return { | |
| "models": str(n_models), | |
| "methods": str(n_models + n_systems), | |
| "methods_label": "models & systems" if n_systems else "models", | |
| "entrants": f"{n_models} models" + (f" and {n_systems} systems" if n_systems else ""), | |
| "verified": str(method[~is_system & verified].nunique()), | |
| } | |
| def hero_facts(data_root: Path) -> dict[str, str]: | |
| """The numbers the TabArena hero quotes, read from the published data and the version history. | |
| Methods are counted in the widest entrant pool, so the hero names everything TabArena covers | |
| rather than whichever pool the page opens on. | |
| """ | |
| df = LBContainer(data_root, Subset(entrants=widest_entrants_key()), "").load_df() | |
| releases, _ = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT) | |
| updated = releases[0].when if releases else "—" | |
| newest = _newest_entrant(releases) | |
| if newest: | |
| fresh = website_texts.HERO_FRESH_NEWEST.format(newest=newest) | |
| else: | |
| fresh = website_texts.HERO_FRESH_UPDATED.format(updated=updated) | |
| return { | |
| "datasets": str(LBContainer(data_root, Subset(), "").n_datasets or "—"), | |
| **_entrant_facts(df), | |
| "releases": str(len(releases)), | |
| "since": f"{releases[-1].date:%b %Y}" if releases else "—", | |
| "updated": updated, | |
| "fresh": fresh, | |
| } | |
| def _dialog(toggle: str, label: str, body: str, *, cls: str, head: str = "") -> str: | |
| """A dialog that shows while its hidden checkbox is checked (see main.taOpenDialog). | |
| ``head`` sits above the body and stays in place while the body scrolls. | |
| """ | |
| return ( | |
| '<div class="ta-dialog-widget">' | |
| f'<input type="checkbox" id="{toggle}" class="ta-dialog-toggle" hidden>' | |
| '<div class="ta-dialog-overlay">' | |
| f'<div class="ta-dialog {cls}" role="dialog" aria-modal="true" aria-label="{html.escape(label)}">' | |
| '<button type="button" class="ta-dialog-x" title="Close" aria-label="Close" ' | |
| 'onclick="taCloseDialogs()">×</button>' | |
| f'{head}<div class="ta-dialog-body">{body}</div>' | |
| "</div></div></div>" | |
| ) | |
| def citation_html(text_id: str, label: str, bibtex: str) -> str: | |
| """A benchmark's BibTeX entry with a button that copies it.""" | |
| return ( | |
| f'<p class="ta-dialog-sub">{html.escape(label)}</p>' | |
| f'<pre class="ta-cite-text" id="{text_id}">{html.escape(bibtex.strip())}</pre>' | |
| f'<button type="button" class="ta-copy" onclick="taCopy(this, \'{text_id}\')">Copy BibTeX</button>' | |
| ) | |
| def metric_definitions_html() -> str: | |
| """The leaderboard's columns, defined as the table's header tooltips define them.""" | |
| rows = "".join( | |
| f"<dt>{html.escape(metric['name'])}</dt><dd>{html.escape(metric['details'])}" | |
| f'<span class="ta-doc-why">Why: {html.escape(metric["why"])}</span></dd>' | |
| for metric in website_texts.METRICS | |
| ) | |
| return f'<h4>{html.escape(website_texts.METRICS_HEADING)}</h4><dl class="ta-doc-defs">{rows}</dl>' | |
| def learn_more( | |
| toggle: str, | |
| label: str, | |
| topics: Sequence[tuple[str, str, str]], | |
| cta: tuple[str, str] | None = None, | |
| hints: dict[str, str] | None = None, | |
| ) -> tuple[str, str]: | |
| """A page's Learn more row and the dialog it opens, for (key, label, HTML) topics. | |
| Every topic is in the page from the start, so a pill opens at once without a round trip to the | |
| server, and the tabs at the top of the dialog switch topic without closing it. Returned apart, | |
| so the row can sit inside a layout the dialog would otherwise add a gap to. ``hints`` maps a | |
| topic to its pill's hover text. ``cta``, as (label, link), ends the row as a highlighted link out, | |
| which also draws the eye to the pills. | |
| """ | |
| # The tour leads the row (main.taStartTour), where a newcomer looks first. | |
| pills = ( | |
| f'<button type="button" class="ta-learn-pill ta-tour-pill" onclick="taStartTour()" ' | |
| f'title="{html.escape(website_texts.TOUR_START_HINT, quote=True)}">' | |
| f'<span aria-hidden="true">▶\ufe0e</span> {html.escape(website_texts.TOUR_START)}</button>' | |
| ) | |
| for key, name, _ in topics: | |
| hint = (hints or {}).get(key) | |
| title = f' title="{html.escape(hint)}"' if hint else "" | |
| pills += ( | |
| f'<button type="button" class="ta-learn-pill"{title} onclick="taOpenDoc(\'{toggle}\', \'{key}\')">' | |
| f"{html.escape(name)}</button>" | |
| ) | |
| if cta: | |
| cta_label, url = cta | |
| pills += ( | |
| f'<a class="ta-learn-cta" href="{url}" target="_blank" rel="noopener noreferrer">{html.escape(cta_label)}' | |
| '<span class="ta-learn-cta-out" aria-hidden="true">↗\ufe0e</span></a>' | |
| ) | |
| tabs = "".join( | |
| f'<button type="button" class="ta-doc-tab" role="tab" data-topic="{key}" ' | |
| f"onclick=\"taShowDoc(this.closest('.ta-dialog'), '{key}')\">{html.escape(name)}</button>" | |
| for key, name, _ in topics | |
| ) | |
| panes = "".join(f'<section class="ta-doc-pane" data-topic="{key}">{body}</section>' for key, _, body in topics) | |
| row = f'<div class="ta-learn"><span class="ta-learn-label">{website_texts.LEARN_MORE_LABEL}</span>{pills}</div>' | |
| head = f'<div class="ta-doc-tabs" role="tablist">{tabs}</div>' | |
| return row, _dialog(toggle, label, panes, cls="ta-dialog-docs", head=head) | |
| def _avatar(login: str, size: int) -> str: | |
| """A maintainer's GitHub avatar, fetched at twice its size for sharp screens.""" | |
| return ( | |
| f'<img class="ta-avatar" src="https://avatars.githubusercontent.com/{login}?s={2 * size}" ' | |
| f'width="{size}" height="{size}" alt="" loading="lazy">' | |
| ) | |
| def _team_dialog(toggle: str, title: str, team: Sequence[tuple[str, str]]) -> str: | |
| """The maintainers, as (name, GitHub login), each linking to their profile, and who to ask what.""" | |
| members = [] | |
| for name, login in team: | |
| primary = login == website_texts.TEAM_PRIMARY_CONTACT | |
| badge = f'<span class="ta-member-badge">{website_texts.TEAM_PRIMARY_LABEL}</span>' if primary else "" | |
| members.append( | |
| f'<a class="ta-member{" is-primary" if primary else ""}" href="https://github.com/{login}" ' | |
| f'target="_blank" rel="noopener noreferrer">{_avatar(login, 44)}' | |
| f'<span class="ta-member-text"><span class="ta-member-name">{html.escape(name)}</span>' | |
| f'<span class="ta-member-handle">@{login}</span></span>{badge}</a>' | |
| ) | |
| body = ( | |
| f"<h2>{html.escape(title)}</h2>" | |
| f'<p class="ta-dialog-sub">{html.escape(website_texts.TEAM_SUB)}</p>' | |
| f'<div class="ta-team">{"".join(members)}</div>' | |
| f'<div class="ta-team-contact">{render_markdown(website_texts.TEAM_CONTACT)}</div>' | |
| ) | |
| return _dialog(toggle, title, body, cls="ta-dialog-team") | |
| def _hero_stat( | |
| value: str, label: str, line: str, hint: str, target: str, *, facts: dict[str, str], docs_toggle: str | |
| ) -> str: | |
| """One selling point: a big value, what it counts and a short line, opening its target. | |
| A numeric value counts up whenever the band opens (the ``ta-count-up`` animation in main.py). | |
| """ | |
| value, label, line, hint = (part.format(**facts) for part in (value, label, line, hint)) | |
| if value.isdigit(): | |
| num = ( | |
| f'<span class="ta-stat-num" data-count style="--ta-count-to: {value}">' | |
| f'<span class="ta-sr-only">{value}</span></span>' | |
| ) | |
| else: | |
| num = f'<span class="ta-stat-num">{html.escape(value)}</span>' | |
| inner = ( | |
| f'{num}<span class="ta-stat-text"><span class="ta-stat-lbl">{html.escape(label)}</span>' | |
| f'<span class="ta-stat-line">{html.escape(line)}</span></span>' | |
| ) | |
| title = html.escape(hint) | |
| if target.startswith("doc:"): | |
| action = f"taOpenDoc('{docs_toggle}', '{target[4:]}')" | |
| return f'<button type="button" class="ta-stat" title="{title}" onclick="{action}">{inner}</button>' | |
| if target == "history": | |
| # The version history is the proof that the benchmark is alive, so its line has a live dot. | |
| action = f"taOpenDialog('{VERSION_HISTORY_TOGGLE}')" | |
| return f'<button type="button" class="ta-stat" data-live title="{title}" onclick="{action}">{inner}</button>' | |
| return f'<a class="ta-stat" href="{target}" target="_blank" rel="noopener noreferrer" title="{title}">{inner}</a>' | |
| def _page_hero( | |
| slug: str, | |
| facts: dict[str, str], | |
| *, | |
| lead: str, | |
| paper_url: str, | |
| citation: tuple[str, str], | |
| cta: tuple[str, str], | |
| team: tuple[str, Sequence[tuple[str, str]]], | |
| hints: dict[str, str], | |
| stats: Sequence[tuple[str, str, str, str, str]], | |
| docs: Sequence[tuple[str, str, str]], | |
| ) -> gr.HTML: | |
| """The top of a benchmark's page: what it is, its links, and the topics to learn more. | |
| "Why this benchmark?" ends the sentence and opens the selling points (``stats``) under it, hidden at | |
| first; they and the Learn more pills' ``hints`` quote ``facts`` read from the data, so the | |
| numbers cannot drift from what the page shows. The citation (``citation`` is its label and BibTeX), the | |
| team (``team`` is its title and members) and the ``docs`` topics open as dialogs rather than | |
| panels on the page, and ``cta`` ends the Learn more row. The hero takes its name from the page's | |
| link in the top bar and its colour from ``slug``. | |
| """ | |
| name = website_texts.TOPBAR_NAV[slug][1] | |
| cite_toggle, team_toggle, docs_toggle = (_hero_toggle(slug, kind) for kind in ("cite", "team", "docs")) | |
| paper_icon, paper_label = website_texts.HERO_PAPER | |
| cite_icon, cite_label = website_texts.HERO_CITE | |
| head = ( | |
| '<span class="ta-split">' | |
| f'<a class="ta-split-a" href="{paper_url}" target="_blank" rel="noopener noreferrer">' | |
| f'<span class="ta-plink-ico" aria-hidden="true">{paper_icon}</span>{html.escape(paper_label)}</a>' | |
| f'<button type="button" class="ta-split-b" onclick="taOpenDialog(\'{cite_toggle}\')">' | |
| f'<span class="ta-plink-ico" aria-hidden="true">{cite_icon}</span>{html.escape(cite_label)}</button>' | |
| "</span>" | |
| ) | |
| avatars = "".join(_avatar(login, 20) for _, login in team[1]) | |
| head += ( | |
| f'<button type="button" class="ta-plink ta-team-chip" onclick="taOpenDialog(\'{team_toggle}\')">' | |
| f'<span class="ta-avatars" aria-hidden="true">{avatars}</span>{html.escape(website_texts.HERO_TEAM_LABEL)}' | |
| "</button>" | |
| ) | |
| band = "".join(_hero_stat(*stat, facts=facts, docs_toggle=docs_toggle) for stat in stats) | |
| why_id = f"ta-{slug}-why" | |
| why = ( | |
| f' <button type="button" class="ta-why" aria-expanded="false" aria-controls="{why_id}" ' | |
| f'onclick="taToggleWhy(this)">{html.escape(website_texts.HERO_WHY)}</button>' | |
| ) | |
| hints = {key: hint.format(**facts) for key, hint in hints.items()} | |
| learn_row, docs_dialog = learn_more(docs_toggle, f"Learn more about {name}", docs, cta=cta, hints=hints) | |
| cite = f"<h2>Cite {html.escape(name)}</h2>" + citation_html(f"{cite_toggle}-bibtex", *citation) | |
| dialogs = _dialog(cite_toggle, f"Cite {name}", cite, cls="ta-dialog-cite") | |
| dialogs += _team_dialog(team_toggle, *team) + docs_dialog | |
| return gr.HTML( | |
| f'<section class="ta-phero" data-page="{slug}">' | |
| f'<div class="ta-phero-head"><p class="ta-phero-lead">{_MARKDOWN.renderInline(lead)}{why}</p>' | |
| f'<div class="ta-phero-links">{head}</div></div>' | |
| f'<div class="ta-stats" id="{why_id}" hidden>{band}</div>' | |
| f"{learn_row}" | |
| f"</section>{dialogs}", | |
| apply_default_css=False, | |
| elem_classes="ta-phero-block", | |
| ) | |
| def make_tabarena_hero(data_root: Path, docs: Sequence[tuple[str, str, str]]) -> gr.HTML: | |
| """The top of the TabArena page, quoting :func:`hero_facts`, with ``docs`` to learn more.""" | |
| return _page_hero( | |
| "tabarena", | |
| hero_facts(data_root), | |
| lead=website_texts.HERO_LEAD, | |
| paper_url=website_texts.HERO_PAPER_URL, | |
| citation=(website_texts.CITATION_BUTTON_LABEL, website_texts.CITATION_BUTTON_TEXT), | |
| cta=website_texts.HERO_CTA, | |
| team=(website_texts.TEAM_TITLE, website_texts.TEAM), | |
| hints=website_texts.HERO_HINTS, | |
| stats=website_texts.HERO_STATS, | |
| docs=docs, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # Page chrome: the top bar and the dialogs it opens | |
| # --------------------------------------------------------------------------- # | |
| #: The hidden checkbox whose state shows the version-history dialog (see main.taOpenDialog). | |
| VERSION_HISTORY_TOGGLE = "ta-vh-toggle" | |
| _RELEASE_HEAD = re.compile(r"^\* (\d{4})/(\d{2})(?:/(\d{2}))?-(v[\d.]+):(.*)$", re.MULTILINE) | |
| _MARKDOWN = MarkdownIt("commonmark") | |
| _ICON_GITHUB = ( | |
| '<svg viewBox="0 0 16 16" width="16" height="16" aria-hidden="true"><path fill="currentColor" d="M8 0C3.58 0 0 ' | |
| "3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23" | |
| "-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07" | |
| "-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 " | |
| "2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 " | |
| "3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58" | |
| '-8-8-8z"/></svg>' | |
| ) | |
| _ICON_STROKE = ( | |
| '<svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="2" ' | |
| 'stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" class="{cls}">{body}</svg>' | |
| ) | |
| _ICON_INFO = _ICON_STROKE.format(cls="ta-ico", body='<circle cx="12" cy="12" r="9"/><path d="M12 11v5M12 8h.01"/>') | |
| _ICON_SUN = _ICON_STROKE.format( | |
| cls="ta-ico ta-ico-sun", | |
| body='<circle cx="12" cy="12" r="4"/><path d="M12 2v2M12 20v2M4.9 4.9l1.4 1.4M17.7 17.7l1.4 1.4M2 12h2M20 12h2' | |
| 'M4.9 19.1l1.4-1.4M17.7 6.3l1.4-1.4"/>', | |
| ) | |
| _ICON_MOON = _ICON_STROKE.format( | |
| cls="ta-ico ta-ico-moon", body='<path d="M21 12.8A9 9 0 1 1 11.2 3a7 7 0 0 0 9.8 9.8z"/>' | |
| ) | |
| _ICON_DATA = _ICON_STROKE.format( | |
| cls="ta-ico", | |
| body='<ellipse cx="12" cy="5" rx="8" ry="3"/><path d="M4 5v14c0 1.7 3.6 3 8 3s8-1.3 8-3V5"/>' | |
| '<path d="M4 12c0 1.7 3.6 3 8 3s8-1.3 8-3"/>', | |
| ) | |
| _ICON_TOUR = _ICON_STROKE.format(cls="ta-ico", body='<circle cx="12" cy="12" r="9"/><path d="M10 8.5v7l6-3.5z"/>') | |
| _ICON_HISTORY = _ICON_STROKE.format( | |
| cls="ta-ico", | |
| body='<path d="M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8"/><path d="M3 3v5h5"/><path d="M12 7v5l4 2"/>', | |
| ) | |
| class Release: | |
| """One entry of the version history.""" | |
| date: datetime.date | |
| #: False for the oldest entries, which name only a month. | |
| has_day: bool | |
| version: str | |
| #: The entry's changes as Markdown, de-indented so its sub-items become a top-level list. | |
| source: str | |
| def when(self) -> str: | |
| return f"{self.date.day} {self.date:%b %Y}" if self.has_day else f"{self.date:%b %Y}" | |
| def parse_version_history(text: str) -> tuple[list[Release], str]: | |
| """Split the version history into its releases, newest first, and the note after them. | |
| The history is one hand-kept Markdown list (the update-leaderboard skill adds to it): one | |
| ``* YYYY/MM/DD-vX.Y.Z:`` item per release with its changes as sub-items. The note is the | |
| paragraph after the last release (the legacy leaderboards), which belongs to none of them. | |
| """ | |
| heads = list(_RELEASE_HEAD.finditer(text)) | |
| releases, note = [], "" | |
| for i, head in enumerate(heads): | |
| body = text[head.end() : heads[i + 1].start() if i + 1 < len(heads) else len(text)] | |
| if i + 1 == len(heads): | |
| cut = re.search(r"^\S", body, re.MULTILINE) | |
| if cut: | |
| body, note = body[: cut.start()], body[cut.start() :] | |
| year, month, day, version, lead = head.groups() | |
| releases.append( | |
| Release( | |
| date=datetime.date(int(year), int(month), int(day or 1)), | |
| has_day=day is not None, | |
| version=version, | |
| source=(lead.strip() + "\n" + textwrap.dedent(body)).strip(), | |
| ) | |
| ) | |
| return releases, note.strip() | |
| def render_markdown(text: str) -> str: | |
| """Markdown to HTML, with outside links opening a new tab (the Space runs in a frame).""" | |
| return _MARKDOWN.render(text).replace('<a href="http', '<a target="_blank" rel="noopener noreferrer" href="http') | |
| # A benchmark's version at the end of its name ("TabArena-v0.1"), which the top bar sets smaller. | |
| _NAV_VERSION = re.compile(r"^(.+?)(-v[\d.]+)$") | |
| def _nav_link(page: LeaderboardPage) -> str: | |
| """One page link of the site's bar, its icon and then its label. | |
| The benchmarks hosted here lead, larger and framed in their own colour; the pages that point | |
| elsewhere are smaller, and the invitation to add a benchmark is dashed. | |
| """ | |
| icon, label, hint = website_texts.TOPBAR_NAV.get(page.slug, ("", page.name, "")) | |
| versioned = _NAV_VERSION.match(label) | |
| if versioned: | |
| # One span, so the link's flex gap does not part the version from the name. | |
| name = ( | |
| f'<span class="ta-nav-lbl">{html.escape(versioned[1])}' | |
| f'<span class="ta-nav-ver">{html.escape(versioned[2])}</span></span>' | |
| ) | |
| else: | |
| name = f'<span class="ta-nav-lbl">{html.escape(label)}</span>' | |
| if page.kind in ("internal", "beyondarena"): | |
| tier = "ta-nav-primary" | |
| else: | |
| tier = "ta-nav-cta" if page.kind == "invite" else "ta-nav-secondary" | |
| return ( | |
| f'<button type="button" class="ta-nav-link {tier}" data-slug="{page.slug}" ' | |
| f'data-tab="{html.escape(page.name, quote=True)}" title="{html.escape(hint, quote=True)}">' | |
| f'<span class="ta-nav-ico" aria-hidden="true">{icon}</span>{name}</button>' | |
| ) | |
| def _release_button(latest: Release) -> str: | |
| """The current version and when it came out; opens the version history.""" | |
| hint = f"{website_texts.VERSION_HISTORY_HINT}. {website_texts.RELEASE_UPDATED_LABEL} {latest.when}." | |
| return ( | |
| f'<button type="button" class="ta-act ta-release" onclick="taOpenDialog(\'{VERSION_HISTORY_TOGGLE}\')" ' | |
| f'title="{hint}" aria-label="{website_texts.VERSION_HISTORY_TITLE}, {latest.version}, {latest.when}">' | |
| f'{_ICON_HISTORY}<span class="ta-release-text"><span class="ta-release-ver">{latest.version}</span>' | |
| f'<span class="ta-release-date">{latest.when}</span></span></button>' | |
| ) | |
| #: After an outside link's label: the arrow, in its text form rather than the emoji. | |
| _NAV_OUT = '<span class="ta-nav-out" aria-hidden="true"> ↗︎</span>' | |
| def _nav_group(key: str) -> str: | |
| """A group's label in the site's bar (website_texts.TOPBAR_GROUPS); its hover says what the group is.""" | |
| label, hint = website_texts.TOPBAR_GROUPS[key] | |
| return ( | |
| f'<span class="ta-nav-grp" title="{html.escape(hint, quote=True)}">' | |
| f'<span class="ta-nav-lbl">{html.escape(label)}</span></span>' | |
| ) | |
| def _outside_links() -> str: | |
| """The benchmarks outside TabArena the bar links to directly, each opening its own site in a new tab.""" | |
| logos = Path(os.path.relpath(LOGO_DIR)) | |
| def logo(file: str) -> str: | |
| # Each benchmark's own mark, in its light and its dark colours; the page's theme shows one. | |
| dark = Path(file).with_stem(Path(file).stem + "-dark") | |
| return "".join( | |
| f'<img class="{cls}" src="/gradio_api/file={(logos / name).as_posix()}" alt="" width="20" height="20">' | |
| for cls, name in (("ta-logo-light", file), ("ta-logo-dark", dark.name)) | |
| ) | |
| links = "".join( | |
| f'<a class="ta-nav-link ta-nav-secondary ta-nav-ext" href="{html.escape(url, quote=True)}" target="_blank" ' | |
| f'rel="noopener noreferrer" title="{html.escape(hint, quote=True)}">' | |
| f'<span class="ta-nav-ico" aria-hidden="true">{logo(file)}</span>' | |
| f'<span class="ta-nav-lbl">{html.escape(label)}{_NAV_OUT}</span></a>' | |
| for file, label, url, hint in website_texts.TOPBAR_OUTSIDE_LINKS | |
| ) | |
| return links | |
| def _top_bar(pages: Sequence[LeaderboardPage], latest: Release | None) -> str: | |
| hosted = [page for page in pages if page.kind in ("internal", "beyondarena")] | |
| others = [page for page in pages if page not in hosted] | |
| # The hosted benchmarks, then two labelled groups (website_texts.TOPBAR_GROUPS): the benchmarks | |
| # built on TabArena (RamanBench) and those outside it (the direct links and the Time-series | |
| # signposts), then the invitation on its own. | |
| # The closed rail shows only the hosted benchmarks and the page on screen; the rest appear when it | |
| # opens (main.py's rail CSS). | |
| built = [page for page in others if page.kind == "external"] | |
| outside = [page for page in others if page.kind not in ("external", "invite")] | |
| links = "".join(_nav_link(page) for page in hosted) | |
| more = "" | |
| if built: | |
| more += _nav_group("built") + "".join(_nav_link(page) for page in built) | |
| if outside or website_texts.TOPBAR_OUTSIDE_LINKS: | |
| more += _nav_group("outside") + _outside_links() + "".join(_nav_link(page) for page in outside) | |
| invite = "".join(_nav_link(page) for page in others if page.kind == "invite") | |
| if invite: | |
| more += f'<span class="ta-nav-sep" aria-hidden="true"></span>{invite}' | |
| if more: | |
| links += f'<div class="ta-nav-more-list">{more}</div>' | |
| home = pages[0] | |
| return ( | |
| '<header class="ta-topbar">' | |
| f'<button type="button" class="ta-home ta-brand" data-slug="{home.slug}" ' | |
| f'data-tab="{html.escape(home.name, quote=True)}" title="{html.escape(website_texts.TOPBAR_TAGLINE_LONG)}">' | |
| f'<img src="{website_texts.TOPBAR_LOGO_URL}" alt="" width="32" height="32">' | |
| '<span class="ta-brand-text">' | |
| f'<span class="ta-brand-name">{website_texts.TOPBAR_BRAND} ' | |
| f'<span class="ta-brand-suffix">{website_texts.TOPBAR_BRAND_SUFFIX}</span></span>' | |
| f'<span class="ta-brand-tag">{html.escape(website_texts.TOPBAR_TAGLINE)}</span>' | |
| "</span></button>" | |
| f'<nav class="ta-nav" aria-label="Benchmarks">{links}</nav>' | |
| '<div class="ta-actions">' | |
| f'<button type="button" class="ta-act ta-tour-btn" onclick="taStartTour()" ' | |
| f'title="{html.escape(website_texts.TOUR_START_HINT, quote=True)}">{_ICON_TOUR}' | |
| f'<span class="ta-act-lbl">{html.escape(website_texts.TOUR_START)}</span></button>' | |
| f'<a class="ta-act" href="{website_texts.TOPBAR_CODE_URL}" target="_blank" rel="noopener noreferrer" ' | |
| f'title="The benchmark code on GitHub">{_ICON_GITHUB}<span class="ta-act-lbl">Code</span></a>' | |
| f'<a class="ta-act" href="{website_texts.TOPBAR_DATA_URL}" target="_blank" rel="noopener noreferrer" ' | |
| f'title="Data Foundry: where the datasets are collected and curated">{_ICON_DATA}' | |
| '<span class="ta-act-lbl">Data</span></a>' | |
| '<button type="button" class="ta-act" onclick="taOpenDialog(\'coi-toggle\')" ' | |
| f'title="Read our conflict-of-interest statement">{_ICON_INFO}<span class="ta-act-lbl">Our COI</span></button>' | |
| f"{_release_button(latest) if latest is not None else ''}" | |
| '<button type="button" class="ta-act ta-theme-toggle" onclick="taToggleTheme()" ' | |
| f'aria-label="Switch to the light theme" title="Switch to the light theme">{_ICON_SUN}{_ICON_MOON}' | |
| f'<span class="ta-act-lbl">{website_texts.TOPBAR_THEME_LABEL}</span></button>' | |
| "</div></header>" | |
| ) | |
| def _version_history_dialog(releases: list[Release], note: str) -> str: | |
| sections = "".join( | |
| '<section class="ta-vh-rel">' | |
| f'<div class="ta-vh-when"><span class="ta-vh-ver">{release.version}</span>' | |
| f'<time datetime="{release.date.isoformat()}">{release.when}</time></div>' | |
| f'<div class="ta-vh-body">{render_markdown(release.source)}</div>' | |
| "</section>" | |
| for release in releases | |
| ) | |
| body = ( | |
| f"<h2>{website_texts.VERSION_HISTORY_TITLE}</h2>" | |
| f'<p class="ta-vh-sub">{website_texts.VERSION_HISTORY_SUB}</p>' | |
| f"{sections}" | |
| f'<div class="ta-vh-note">{render_markdown(note)}</div>' | |
| ) | |
| return _dialog(VERSION_HISTORY_TOGGLE, website_texts.VERSION_HISTORY_TITLE, body, cls="ta-dialog-vh") | |
| def make_page_chrome(pages: Sequence[LeaderboardPage]) -> None: | |
| """The site's bar: benchmark links, the current release, global actions, dialogs. | |
| A rail down the left edge on a wide screen and a bar above the page on a narrow one (main.py's | |
| top-bar CSS), from the same markup. | |
| One block for all of it, since every extra ``gr.HTML`` adds a layout gap. The release shown | |
| and the version-history dialog are read from ``website_texts.VERSION_HISTORY_BUTTON_TEXT``, | |
| so a release recorded there shows up here without a second edit. | |
| """ | |
| releases, note = parse_version_history(website_texts.VERSION_HISTORY_BUTTON_TEXT) | |
| gr.HTML( | |
| _top_bar(pages, releases[0] if releases else None) | |
| + website_texts.COI_HTML | |
| + _version_history_dialog(releases, note), | |
| apply_default_css=False, | |
| ) | |
| # --------------------------------------------------------------------------- # | |
| # BeyondArena components | |
| # | |
| # BeyondArena reuses the per-subset leaderboard table (make_leaderboard) and the | |
| # figure grid (make_figure_grid) unchanged; only the hero's copy, the | |
| # cross-subset overview (an image, not the TabArena HTML heatmap) and the | |
| # per-subset figure set differ (no HPO tuning-trajectory figure — BeyondArena is | |
| # evaluated on a single `core2k` protocol). | |
| # --------------------------------------------------------------------------- # | |
| def beyond_hero_facts(data_root: Path) -> dict[str, str]: | |
| """The numbers the BeyondArena hero quotes, read from its published data. | |
| The subsets are the selector's filter chips, every axis's "all" aside; they combine freely. | |
| """ | |
| lb = LBContainer(data_root, BeyondSubset(), "") | |
| return { | |
| "datasets": str(lb.n_datasets or "—"), | |
| "subsets": str(len(BEYOND_SUBSET_LABELS)), | |
| **_entrant_facts(lb.load_df()), | |
| } | |
| def make_beyond_hero(data_root: Path, docs: Sequence[tuple[str, str, str]]) -> gr.HTML: | |
| """The top of the BeyondArena page, quoting :func:`beyond_hero_facts`, with ``docs`` to learn more.""" | |
| return _page_hero( | |
| "beyondarena", | |
| beyond_hero_facts(data_root), | |
| lead=website_texts.BEYOND_HERO_LEAD, | |
| paper_url=website_texts.BEYOND_HERO_PAPER_URL, | |
| citation=(website_texts.BEYOND_CITATION_BUTTON_LABEL, website_texts.BEYOND_CITATION_BUTTON_TEXT), | |
| cta=website_texts.BEYOND_HERO_CTA, | |
| team=(website_texts.BEYOND_TEAM_TITLE, website_texts.BEYOND_TEAM), | |
| hints=website_texts.BEYOND_HERO_HINTS, | |
| stats=website_texts.BEYOND_HERO_STATS, | |
| docs=docs, | |
| ) | |
| def make_beyond_overview_figure(data_root: Path, include_leakage: bool = True) -> None: | |
| """The cross-subset overview: best Elo per model / per family across every subset. | |
| Unlike TabArena's HTML heatmap, BeyondArena's overview is the ``plot_subset_results`` image | |
| (``result_plots/per_model_elo`` + ``per_family_elo``). Missing images are skipped gracefully. | |
| ``include_leakage`` off reads their twins computed without the methods with a leakage warning | |
| (``result_plots_noleak/``); on, a note names those methods, since the images cannot mark them. | |
| """ | |
| result_dir = Path(data_root) / ("result_plots" if include_leakage else "result_plots" + NO_LEAKAGE_SUFFIX) | |
| images = [ | |
| ("per_family_elo", "Best Elo per model type (family) across subsets"), | |
| ("per_model_elo", "Best Elo per model across subsets"), | |
| ] | |
| shown = False | |
| for name, label in images: | |
| if (result_dir / f"{name}.png").exists() or (result_dir / f"{name}.png.zip").exists(): | |
| # No fixed height: the per-model figure is about 6:1, and a fixed box letterboxed it. | |
| gr.Image( | |
| unzip_png(result_dir, name), | |
| label=label, | |
| show_label=True, | |
| elem_classes="ta-view-card", | |
| ) | |
| shown = True | |
| if not shown: | |
| gr.Markdown("_The cross-subset overview figure is not available yet._", elem_classes="markdown-text") | |
| elif include_leakage: | |
| try: | |
| flagged = leakage_methods(LBContainer(data_root, BeyondSubset(), "").load_df()) | |
| except FileNotFoundError: | |
| flagged = [] | |
| if flagged: | |
| gr.Markdown( | |
| website_texts.LEAKAGE_OVERVIEW_NOTE.format(names=", ".join(flagged)), | |
| elem_classes=["markdown-text", "ta-lk-note"], | |
| ) | |