| const leaderboardConfig = { |
| textual: { |
| label: "Textual Memory", |
| categories: [ |
| ["Overall", "Aggregate performance across the fixed textual evaluation suite."], |
| ["Explicit Fact Recall", "Facts, attributes, sources, and entities recalled from long histories."], |
| ["Compositional Inference", "Relations and multi-hop evidence combined across memory fragments."], |
| ["Temporal & Event Reasoning", "Dates, ordering, intervals, and evolving states over time."], |
| ["Memory Governance", "Updates, conflict resolution, deletion, and forgetting."], |
| ["Personalization & Care", "Preferences, background, wellbeing context, and appropriate adaptation."], |
| ["Context Learning & Execution", "Rules, procedures, domain constraints, and format execution."], |
| ["Safety & Privacy", "Evidence boundaries, robust abstention, and minimal disclosure."], |
| ], |
| columns: ["Rank", "System", "Overall", "Fact Recall", "Composition", "Temporal", "Governance", "Personalization", "Execution", "Safety", "Total Time", "Write Time", "Search Time", "Return Size"], |
| }, |
| coding: { |
| label: "Coding Memory", |
| categories: [ |
| ["Overall", "Aggregate performance across 150 coding-memory tasks, ranked by Task Solve (%)."], |
| ["New Feature", "Performance across 51 new-feature implementation tasks."], |
| ["Bug Fix", "Performance across 99 bug-fixing tasks."], |
| ], |
| columns: ["Rank", "System", "Task Solve (%)", "New Feature", "Bug Fix", "Total Time", "Write Time", "Search Time", "Return Size"], |
| }, |
| }; |
|
|
| let textualLeaderboardData = null; |
| let codingLeaderboardData = null; |
|
|
| const textualOverallScoreFields = [ |
| { field: "overall", label: "Overall" }, |
| { field: "factRecall", label: "Fact Recall" }, |
| { field: "composition", label: "Composition" }, |
| { field: "temporal", label: "Temporal" }, |
| { field: "governance", label: "Governance" }, |
| { field: "personalization", label: "Personalization" }, |
| { field: "execution", label: "Execution" }, |
| { field: "safety", label: "Safety" }, |
| { field: "totalTime", label: "Total Time", type: "speed" }, |
| { field: "writeTime", label: "Write Time", type: "speed" }, |
| { field: "searchTime", label: "Search Time", type: "speed" }, |
| { field: "returnSize", label: "Return Size", type: "volume" }, |
| ]; |
|
|
| const textualStatusFields = textualOverallScoreFields.filter(({ type }) => type === "speed" || type === "volume"); |
|
|
| const codingOverallScoreFields = [ |
| { field: "taskSolve", label: "Task Solve (%)" }, |
| { field: "newFeature", label: "New Feature" }, |
| { field: "bugFix", label: "Bug Fix" }, |
| ...textualStatusFields, |
| ]; |
|
|
| const textualHeaderJumpTargets = { |
| "Overall": 0, |
| "Fact Recall": 1, |
| "Composition": 2, |
| "Temporal": 3, |
| "Governance": 4, |
| "Personalization": 5, |
| "Execution": 6, |
| "Safety": 7, |
| }; |
|
|
| const codingHeaderJumpTargets = { |
| "Task Solve (%)": 0, |
| "New Feature": 1, |
| "Bug Fix": 2, |
| }; |
|
|
| function formatScore(value) { |
| return Number.isFinite(value) ? (value * 100).toFixed(1) : "--"; |
| } |
|
|
| function formatSystemName(name) { |
| const aliases = { |
| }; |
| const cleanedName = name.replace(/\s*\(topk=100\)/gi, "").trim(); |
| return aliases[cleanedName] ?? cleanedName; |
| } |
|
|
| const academicGithubLinks = { |
| "LightMem": "https://github.com/zjunlp/LightMem", |
| "Lightmem-StructMem": "https://github.com/zjunlp/LightMem", |
| "LightMem-Ego-lite": "https://github.com/Crosser-XDU/LightMem-Ego-text-service", |
| "M2A": "https://github.com/Crosser-XDU/M2A-text-service", |
| "InvMem": "https://github.com/wenxiaof345-ctrl/vanilla-rag-memory", |
| "Refind": "https://github.com/imlrz/ReFind", |
| "ActiveMemoryIndex": "https://github.com/linxuhao/ActiveMemoryIndex.git", |
| "Hybrid Search v2.0": "https://github.com/cydd-1972/hybrid_search", |
| "Hybrid Episodic Memory": "https://github.com/tlysanhuo/agent-memory-challenge", |
| "LLLMemoryAgent": "https://github.com/llLAlisa/memory-agent-submission", |
| "Chronicle": "https://github.com/simple-boy/Chronicle-Memory", |
| "FlowGrid_AML_Retriever_1_0_0": "https://github.com/dlxeva/flowgrid-aml-retriever", |
| "aml-memory-mvp": "https://github.com/0xboyu/aml-memory-mvp", |
| "Memoria_v2": "https://github.com/lin-guan/aml-rag-v2", |
| "ChronoHybridMem": "https://github.com/Tin11Mn/chrono-hybrid-mem", |
| }; |
|
|
| function getAcademicGithubLink(systemName) { |
| if (state.division !== "academic") return null; |
| const cleanedName = systemName.replace(/\s*\(topk=100\)/gi, "").trim(); |
| return academicGithubLinks[cleanedName] ?? null; |
| } |
|
|
| function renderGithubIconLink(url, systemName) { |
| const link = document.createElement("a"); |
| link.href = url; |
| link.target = "_blank"; |
| link.rel = "noreferrer"; |
| link.className = "github-icon-link"; |
| link.setAttribute("aria-label", `Open ${systemName} GitHub repository`); |
| link.title = "GitHub"; |
| link.innerHTML = ` |
| <svg viewBox="0 0 16 16" aria-hidden="true" focusable="false"> |
| <path d="M8 0.2A8 8 0 0 0 5.47 15.8c0.4 0.07 0.55-0.17 0.55-0.38v-1.36c-2.22 0.48-2.69-1.07-2.69-1.07-0.36-0.92-0.88-1.17-0.88-1.17-0.72-0.49 0.05-0.48 0.05-0.48 0.8 0.06 1.22 0.82 1.22 0.82 0.71 1.22 1.87 0.87 2.32 0.66 0.07-0.52 0.28-0.87 0.51-1.07-1.77-0.2-3.64-0.89-3.64-3.95 0-0.87 0.31-1.58 0.82-2.14-0.08-0.2-0.36-1.01 0.08-2.11 0 0 0.67-0.21 2.2 0.82A7.5 7.5 0 0 1 8 4.1c0.68 0 1.36 0.09 2 0.27 1.52-1.03 2.19-0.82 2.19-0.82 0.44 1.1 0.16 1.91 0.08 2.11 0.51 0.56 0.82 1.27 0.82 2.14 0 3.07-1.87 3.75-3.65 3.95 0.29 0.25 0.54 0.73 0.54 1.48v2.19c0 0.21 0.15 0.46 0.55 0.38A8 8 0 0 0 8 0.2Z" /> |
| </svg>`; |
| return link; |
| } |
|
|
| function getCateTooltip(cate, division) { |
| if (cate === "Submitted (API)") { |
| return "Submitted directly by the original authors/developers with their provided API."; |
| } |
| if (cate === "Submitted (repo)") { |
| return "Submitted directly by the original authors/developers with their provided runtime parameters and environments."; |
| } |
| if (cate === "Evaluated") { |
| return division === "commercial" |
| ? "Evaluated by the leaderboard committee using the method's public API." |
| : "Evaluated by the leaderboard committee using the method's public GitHub repository, or paper reimplementation (see repository for details)."; |
| } |
| return ""; |
| } |
|
|
| let activeBadgeTooltip = null; |
|
|
| function hideBadgeTooltip() { |
| if (activeBadgeTooltip) { |
| activeBadgeTooltip.remove(); |
| activeBadgeTooltip = null; |
| } |
| } |
|
|
| function showBadgeTooltip(target) { |
| const text = target.dataset.tooltip; |
| if (!text) return; |
| hideBadgeTooltip(); |
| const tooltip = document.createElement("span"); |
| tooltip.className = "badge-floating-tooltip"; |
| tooltip.textContent = text; |
| document.body.append(tooltip); |
|
|
| const rect = target.getBoundingClientRect(); |
| const margin = 10; |
| const tooltipRect = tooltip.getBoundingClientRect(); |
| const left = Math.min( |
| Math.max(rect.left, margin), |
| window.innerWidth - tooltipRect.width - margin, |
| ); |
| const top = Math.max(rect.top - tooltipRect.height - 10, margin); |
| tooltip.style.left = `${left}px`; |
| tooltip.style.top = `${top}px`; |
| activeBadgeTooltip = tooltip; |
| } |
|
|
| function attachBadgeTooltip(target) { |
| target.addEventListener("mouseenter", () => showBadgeTooltip(target)); |
| target.addEventListener("mouseleave", hideBadgeTooltip); |
| target.addEventListener("focus", () => showBadgeTooltip(target)); |
| target.addEventListener("blur", hideBadgeTooltip); |
| } |
|
|
| function getScoreTier(value, tiers) { |
| return tiers.findIndex((score) => score === value) + 1; |
| } |
|
|
| function getColumnTiers(rows, field) { |
| return [...new Set(rows.map((entry) => entry[field]).filter(Number.isFinite))] |
| .sort((a, b) => b - a) |
| .slice(0, 3); |
| } |
|
|
| function getCompetitionRanks(rows, field) { |
| const ranks = []; |
| let lastScore = null; |
| let lastRank = 0; |
|
|
| rows.forEach((entry, index) => { |
| const score = entry[field]; |
| if (!Number.isFinite(score)) { |
| ranks.push(index + 1); |
| lastScore = score; |
| lastRank = index + 1; |
| return; |
| } |
|
|
| if (index === 0 || score !== lastScore) { |
| lastRank = index + 1; |
| } |
|
|
| ranks.push(lastRank); |
| lastScore = score; |
| }); |
|
|
| return ranks; |
| } |
|
|
| function renderScoreCell(value, tier) { |
| const cell = document.createElement("span"); |
| cell.className = "score-cell"; |
| const score = document.createElement("b"); |
| score.className = tier > 0 ? `score-value score-rank-${tier}` : "score-value"; |
| score.textContent = formatScore(value); |
| cell.append(score); |
| return cell; |
| } |
|
|
| function renderStatusCell(value, type) { |
| const cell = document.createElement("span"); |
| cell.className = `score-cell ${type === "volume" ? "volume-cell" : "speed-cell"}`; |
| const status = document.createElement("b"); |
| const normalized = typeof value === "string" ? value : "--"; |
| const statusMap = { |
| "快速": { className: "speed-fast", label: "Fast" }, |
| Fast: { className: "speed-fast", label: "Fast" }, |
| "中速": { className: "speed-medium", label: "Medium" }, |
| Medium: { className: "speed-medium", label: "Medium" }, |
| "慢速": { className: "speed-slow", label: "Slow" }, |
| Slow: { className: "speed-slow", label: "Slow" }, |
| Compact: { className: "volume-compact", label: "Compact" }, |
| Balanced: { className: "volume-balanced", label: "Balanced" }, |
| Verbose: { className: "volume-verbose", label: "Verbose" }, |
| }; |
| const state = statusMap[normalized] || { className: "speed-unknown", label: "--" }; |
| if (type === "volume") { |
| status.className = `speed-value volume-value ${state.className}`; |
| status.textContent = state.label; |
| } else { |
| status.className = `speed-inline ${state.className}`; |
| status.innerHTML = `<span class="speed-divider" aria-hidden="true">|</span><span>${state.label}</span>`; |
| } |
| cell.append(status); |
| return cell; |
| } |
|
|
| function renderTextCell(value) { |
| const cell = document.createElement("span"); |
| cell.className = "text-cell"; |
| cell.textContent = value || "--"; |
| return cell; |
| } |
|
|
| function renderMetricCell(entry, metric, tiers) { |
| if (metric.type === "text") { |
| return renderTextCell(entry[metric.field]); |
| } |
| if (metric.type === "speed" || metric.type === "volume") { |
| return renderStatusCell(entry[metric.field], metric.type); |
| } |
| return renderScoreCell(entry[metric.field], getScoreTier(entry[metric.field], tiers)); |
| } |
|
|
| function getTextualView() { |
| if (!textualLeaderboardData) { |
| return { rows: [], metrics: textualOverallScoreFields }; |
| } |
|
|
| if (state.category === 0) { |
| return { |
| rows: textualLeaderboardData.overall[state.division] ?? [], |
| metrics: textualOverallScoreFields, |
| }; |
| } |
|
|
| const categoryData = textualLeaderboardData.categories[state.division]?.[String(state.category)]; |
| if (!categoryData) return { rows: [], metrics: [] }; |
|
|
| return { |
| rows: categoryData.rows.map((entry) => ({ |
| ...entry, |
| categoryScore: entry.categoryScore, |
| ...entry.leaves, |
| })), |
| metrics: [ |
| ...categoryData.columns.map((label, index) => ({ |
| field: index === 0 ? "categoryScore" : label, |
| label, |
| })), |
| ...textualStatusFields, |
| ], |
| }; |
| } |
|
|
| function getCodingView() { |
| if (!codingLeaderboardData) { |
| return { rows: [], metrics: codingOverallScoreFields }; |
| } |
|
|
| if (state.category === 0) { |
| return { |
| rows: codingLeaderboardData.overall[state.division] ?? [], |
| metrics: codingOverallScoreFields, |
| }; |
| } |
|
|
| const categoryData = codingLeaderboardData.categories[state.division]?.[String(state.category)]; |
| if (!categoryData) return { rows: [], metrics: [] }; |
|
|
| return { |
| rows: categoryData.rows, |
| metrics: [ |
| ...categoryData.columns.map((label, index) => ({ |
| field: index === 0 ? "categoryScore" : label, |
| label, |
| })), |
| ...textualStatusFields, |
| ], |
| }; |
| } |
|
|
| function renderLeaderboardRows(view) { |
| const table = document.getElementById("leaderboard-table"); |
| const { rows, metrics } = view; |
| table.replaceChildren(); |
| table.style.setProperty("--metric-count", String(metrics.length)); |
|
|
| const primaryScoreField = metrics[0]?.field; |
| const displayRanks = primaryScoreField ? getCompetitionRanks(rows, primaryScoreField) : []; |
| const scoreTiersByField = Object.fromEntries( |
| metrics.map(({ field, type }) => [field, type === "speed" ? [] : getColumnTiers(rows, field)]), |
| ); |
|
|
| rows.forEach((entry, index) => { |
| const displayRank = displayRanks[index] ?? entry.rank ?? index + 1; |
| const row = document.createElement("article"); |
| row.className = displayRank <= 3 ? "leaderboard-row is-podium-row" : "leaderboard-row"; |
|
|
| const rank = document.createElement("span"); |
| rank.className = displayRank <= 3 ? `rank-cell rank-medal rank-medal-${displayRank}` : "rank-cell"; |
| rank.textContent = displayRank === 1 ? "🥇" : displayRank === 2 ? "🥈" : displayRank === 3 ? "🥉" : String(displayRank).padStart(2, "0"); |
| rank.setAttribute("aria-label", `Rank ${displayRank}`); |
|
|
| const system = document.createElement("span"); |
| system.className = "system-cell"; |
| const link = entry.githubUrl || getAcademicGithubLink(entry.system); |
| const name = document.createElement("strong"); |
| name.textContent = formatSystemName(entry.system); |
| system.append(name); |
| const meta = document.createElement("span"); |
| meta.className = "system-meta"; |
| if (link) meta.append(renderGithubIconLink(link, entry.system)); |
| if (entry.cate) { |
| const cate = document.createElement("small"); |
| cate.className = "system-cate"; |
| cate.textContent = entry.cate; |
| const tooltip = getCateTooltip(entry.cate, state.division); |
| if (tooltip) { |
| cate.dataset.tooltip = tooltip; |
| cate.tabIndex = 0; |
| cate.setAttribute("aria-label", `${entry.cate}: ${tooltip}`); |
| attachBadgeTooltip(cate); |
| } |
| meta.append(cate); |
| } |
| if (entry.version) { |
| const version = document.createElement("small"); |
| version.className = "system-version"; |
| version.textContent = entry.version; |
| version.dataset.tooltip = entry.version; |
| version.tabIndex = 0; |
| version.setAttribute("aria-label", `Version ${entry.version}`); |
| attachBadgeTooltip(version); |
| meta.append(version); |
| } |
| if (meta.childElementCount) system.append(meta); |
|
|
| row.append( |
| rank, |
| system, |
| ...metrics.map((metric) => renderMetricCell(entry, metric, scoreTiersByField[metric.field])), |
| ); |
| table.append(row); |
| }); |
| } |
|
|
| const state = { |
| view: window.location.hash === "#leaderboard" ? "leaderboard" : "home", |
| track: "textual", |
| division: "academic", |
| category: 0, |
| }; |
|
|
| function setView(view, updateHash = true) { |
| state.view = view; |
| document.querySelectorAll("[data-view]").forEach((element) => { |
| const active = element.dataset.view === view; |
| element.hidden = !active; |
| element.classList.toggle("is-active", active); |
| }); |
|
|
| document.querySelectorAll(".nav-link[data-view-link]").forEach((button) => { |
| button.classList.toggle("is-active", button.dataset.viewLink === view); |
| }); |
|
|
| if (updateHash) { |
| history.pushState(null, "", view === "leaderboard" ? "#leaderboard" : "#home"); |
| } |
|
|
| window.scrollTo({ top: 0, behavior: "smooth" }); |
| } |
|
|
| function renderLeaderboard() { |
| const config = leaderboardConfig[state.track]; |
| const [category, description] = config.categories[state.category]; |
| const isOverall = state.category === 0; |
| const activeView = state.track === "textual" ? getTextualView() : getCodingView(); |
| const columns = ["Rank", "System", ...activeView.metrics.map(({ label }) => label)]; |
| const headerJumpTargets = state.track === "textual" ? textualHeaderJumpTargets : codingHeaderJumpTargets; |
|
|
| const divisionLabel = state.division === "academic" ? "Academic Methods" : "Commercial Products"; |
| document.getElementById("result-kicker").textContent = `${config.label} · ${divisionLabel}`; |
| document.getElementById("result-title").textContent = isOverall ? "Overall All Results" : category; |
| document.getElementById("result-description").textContent = isOverall ? "" : description; |
| const resultNote = document.getElementById("result-note"); |
| if (resultNote) resultNote.hidden = state.track !== "textual" || state.division !== "academic"; |
| const commercialAck = document.getElementById("commercial-ack"); |
| if (commercialAck) commercialAck.hidden = state.track !== "textual" || state.division !== "commercial"; |
|
|
| const categoryTabs = document.getElementById("category-tabs"); |
| categoryTabs.replaceChildren(); |
| config.categories.forEach(([label], index) => { |
| const button = document.createElement("button"); |
| button.type = "button"; |
| button.role = "tab"; |
| button.textContent = label; |
| button.classList.toggle("is-active", index === state.category); |
| button.setAttribute("aria-selected", String(index === state.category)); |
| button.addEventListener("click", () => { |
| state.category = index; |
| renderLeaderboard(); |
| }); |
| categoryTabs.append(button); |
| }); |
|
|
| const tableHeader = document.getElementById("table-header"); |
| tableHeader.style.setProperty("--metric-count", String(columns.length - 2)); |
| tableHeader.replaceChildren( |
| ...columns.map((label, index) => { |
| const cell = document.createElement(index >= 2 && headerJumpTargets[label] !== undefined ? "button" : "span"); |
| cell.textContent = label; |
| if (index >= 2 && headerJumpTargets[label] !== undefined) { |
| cell.type = "button"; |
| cell.className = "header-link"; |
| cell.addEventListener("click", () => { |
| state.category = headerJumpTargets[label]; |
| document.querySelectorAll("[data-track]").forEach((trackButton) => { |
| const active = trackButton.dataset.track === state.track; |
| trackButton.classList.toggle("is-active", active); |
| trackButton.setAttribute("aria-selected", String(active)); |
| }); |
| document.querySelectorAll("[data-division]").forEach((divisionButton) => { |
| const active = divisionButton.dataset.division === state.division; |
| divisionButton.classList.toggle("is-active", active); |
| divisionButton.setAttribute("aria-selected", String(active)); |
| }); |
| renderLeaderboard(); |
| document.getElementById("leaderboard-table")?.scrollIntoView({ behavior: "smooth", block: "start" }); |
| }); |
| } |
| return cell; |
| }), |
| ); |
|
|
| renderLeaderboardRows(activeView); |
| } |
|
|
| document.querySelectorAll("[data-view-link]").forEach((element) => { |
| element.addEventListener("click", (event) => { |
| event.preventDefault(); |
| setView(element.dataset.viewLink); |
| }); |
| }); |
|
|
| document.querySelectorAll("[data-open-track]").forEach((button) => { |
| button.addEventListener("click", () => { |
| state.track = button.dataset.openTrack; |
| state.category = 0; |
| document.querySelectorAll("[data-track]").forEach((trackButton) => { |
| const active = trackButton.dataset.track === state.track; |
| trackButton.classList.toggle("is-active", active); |
| trackButton.setAttribute("aria-selected", String(active)); |
| }); |
| renderLeaderboard(); |
| setView("leaderboard"); |
| }); |
| }); |
|
|
| document.querySelectorAll("[data-track]").forEach((button) => { |
| button.addEventListener("click", () => { |
| state.track = button.dataset.track; |
| state.category = 0; |
| document.querySelectorAll("[data-track]").forEach((trackButton) => { |
| const active = trackButton === button; |
| trackButton.classList.toggle("is-active", active); |
| trackButton.setAttribute("aria-selected", String(active)); |
| }); |
| renderLeaderboard(); |
| }); |
| }); |
|
|
| document.querySelectorAll("[data-division]").forEach((button) => { |
| button.addEventListener("click", () => { |
| state.division = button.dataset.division; |
| document.querySelectorAll("[data-division]").forEach((divisionButton) => { |
| const active = divisionButton === button; |
| divisionButton.classList.toggle("is-active", active); |
| divisionButton.setAttribute("aria-selected", String(active)); |
| }); |
| renderLeaderboard(); |
| }); |
| }); |
|
|
| window.addEventListener("popstate", () => { |
| setView(window.location.hash === "#leaderboard" ? "leaderboard" : "home", false); |
| }); |
|
|
| async function loadTextualLeaderboardData() { |
| if (window.__LEADERBOARD_DATA__) { |
| textualLeaderboardData = window.__LEADERBOARD_DATA__; |
| return; |
| } |
| const response = await fetch("./data/generated/leaderboard_data.json", { cache: "no-store" }); |
| if (!response.ok) { |
| throw new Error(`Failed to load leaderboard data: ${response.status}`); |
| } |
| textualLeaderboardData = await response.json(); |
| } |
|
|
| async function loadCodingLeaderboardData() { |
| if (window.__CODE_LEADERBOARD_DATA__) { |
| codingLeaderboardData = window.__CODE_LEADERBOARD_DATA__; |
| return; |
| } |
| const response = await fetch("./data/generated/code_leaderboard_data.json", { cache: "no-store" }); |
| if (!response.ok) { |
| throw new Error(`Failed to load coding leaderboard data: ${response.status}`); |
| } |
| codingLeaderboardData = await response.json(); |
| } |
|
|
| async function initApp() { |
| try { |
| await Promise.all([loadTextualLeaderboardData(), loadCodingLeaderboardData()]); |
| } catch (error) { |
| console.error(error); |
| const table = document.getElementById("leaderboard-table"); |
| if (table) { |
| table.replaceChildren(); |
| const message = document.createElement("article"); |
| message.className = "leaderboard-row"; |
| message.textContent = "Leaderboard data failed to load."; |
| table.append(message); |
| } |
| } |
| renderLeaderboard(); |
| setView(state.view, false); |
| } |
|
|
| initApp(); |
|
|
