toolcall-lint / web-app.js
Meryem Sakin
Fix contract checks that passed templates dropping every tool call
25ebcce
Raw History Blame Contribute Delete
32.8 kB
import {
AutoTokenizer,
env,
} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0";
env.allowLocalModels = false;
env.useBrowserCache = true;
const CONTRACT_VERSION = "2026-08-mvp2-js1";
const TOOL_RESULT_MARKER = "TOOL_RESULT_7f3a";
// Argument values must be attributable to the assistant tool call alone. They are
// deliberately absent from the tool schema below and from every prompt string, so
// finding one in a rendering proves the tool call itself survived.
const ARG_SENTINEL_A = "TOOLCALL_ARG_A_81d20";
const ARG_SENTINEL_B = "TOOLCALL_ARG_B_c93af";
// Prompts must not echo any string a check searches for.
const NEUTRAL_PROMPT = "Use the available tools.";
// Authoritative tier for every check; mirrors CHECK_TIERS in src/toolcall_lint/checks.py.
// `content_none_equivalence` is proposed rather than required because the maintainer
// clarification it encodes (transformers PR #45422) has not merged.
const CHECK_TIERS = {
dict_arguments: "normative",
empty_content: "normative",
parallel_calls: "normative",
tool_response: "normative",
content_none_equivalence: "proposed",
string_arguments: "diagnostic",
prefix_preservation: "diagnostic",
jinja_portability: "diagnostic",
};
const STATUS_ICON = { pass: "✓", warn: "!", fail: "×", skip: "–", error: "×" };
const CHECK_TITLES = {
dict_arguments: "Dictionary tool arguments",
string_arguments: "String argument handling",
content_none_equivalence: "Missing content equivalence",
empty_content: "Empty assistant content",
parallel_calls: "Parallel tool calls",
tool_response: "Tool response preservation",
prefix_preservation: "Training prefix preservation",
jinja_portability: "Cross-engine Jinja portability",
};
const TOOLS = [
{
type: "function",
function: {
name: "get_weather",
description: "Get the weather for a city.",
parameters: {
type: "object",
properties: { city: { type: "string" } },
required: ["city"],
additionalProperties: false,
},
},
},
{
type: "function",
function: {
name: "get_time",
description: "Get the local time for a city.",
parameters: {
type: "object",
properties: { city: { type: "string" } },
required: ["city"],
additionalProperties: false,
},
},
},
];
const form = document.querySelector("#inspect-form");
const modelInput = document.querySelector("#model-id");
const runButton = document.querySelector("#run-button");
const runState = document.querySelector("#run-state");
const runStateLabel = document.querySelector("#run-state-label");
const runStatePercent = document.querySelector("#run-state-percent");
const runStateDetail = document.querySelector("#run-state-detail");
const progressBar = document.querySelector("#progress-bar");
const liveError = document.querySelector("#live-error");
const liveResults = document.querySelector("#live-results");
const baselineBody = document.querySelector("#baseline-body");
const baselineSummary = document.querySelector("#baseline-summary");
const modelSearch = document.querySelector("#model-search");
const statusFilter = document.querySelector("#status-filter");
const tableEmpty = document.querySelector("#table-empty");
const dialog = document.querySelector("#detail-dialog");
const dialogTitle = document.querySelector("#dialog-title");
const dialogContent = document.querySelector("#dialog-content");
let baselineDataset = null;
let lastLiveReport = null;
function toolCall(name, city, argumentsAsString = false, callId = "call_a") {
const args = argumentsAsString ? JSON.stringify({ city }) : { city };
return {
id: callId,
type: "function",
function: { name, arguments: args },
};
}
function assistantMessage({ content = "", includeContent = true, calls = null } = {}) {
const message = {
role: "assistant",
tool_calls: calls || [toolCall("get_weather", ARG_SENTINEL_A)],
};
if (includeContent) message.content = content;
return message;
}
function safeRender(tokenizer, messages) {
try {
const value = tokenizer.apply_chat_template(messages, {
tools: TOOLS,
tokenize: false,
add_generation_prompt: false,
});
return { value, error: null };
} catch (error) {
return { value: null, error: `${error?.name || "Error"}: ${error?.message || String(error)}` };
}
}
function result(checkId, status, summary, evidence = {}) {
return {
check_id: checkId,
title: CHECK_TITLES[checkId],
status,
summary,
evidence,
tier: CHECK_TIERS[checkId],
};
}
// Render the same conversation with and without `calls`. The control rendering
// isolates the output attributable to the assistant tool call, so a template that
// echoes tool schemas or prompt text cannot satisfy a check by accident.
function differential(tokenizer, calls, { content = "", includeContent = true } = {}) {
const userTurn = { role: "user", content: NEUTRAL_PROMPT };
const probe = safeRender(tokenizer, [userTurn, assistantMessage({ content, includeContent, calls })]);
if (probe.error) return { probe: null, control: null, error: probe.error };
const controlMessage = { role: "assistant" };
if (includeContent) controlMessage.content = content;
const control = safeRender(tokenizer, [userTurn, controlMessage]);
return { probe: probe.value, control: control.error ? null : control.value, error: null };
}
// The argument value is a sentinel that exists nowhere else in the input, so its
// presence is direct evidence. The function name also appears in the tool schema,
// so it only counts when the tool call adds an occurrence the control lacks.
function callSurvived(probe, control, name, sentinel) {
const finding = {
argument_sentinel_present: probe.includes(sentinel),
name_occurrences_with_call: countOf(probe, name),
};
if (control === null) {
finding.differential = "unavailable";
finding.survived = finding.argument_sentinel_present;
return finding;
}
finding.name_occurrences_without_call = countOf(control, name);
finding.name_attributable_to_call = countOf(probe, name) > countOf(control, name);
finding.argument_sentinel_leaked_into_control = control.includes(sentinel);
finding.survived =
finding.argument_sentinel_present &&
finding.name_attributable_to_call &&
!finding.argument_sentinel_leaked_into_control;
return finding;
}
function commonPrefixLength(left, right) {
const limit = Math.min(left.length, right.length);
for (let index = 0; index < limit; index += 1) {
if (left[index] !== right[index]) return index;
}
return limit;
}
function diffEvidence(left, right, radius = 100) {
const index = commonPrefixLength(left, right);
const start = Math.max(0, index - radius);
const end = index + radius;
return {
first_difference: index,
canonical_window: left.slice(start, end),
comparison_window: right.slice(start, end),
};
}
function checkDictArguments(tokenizer) {
const { probe, control, error } = differential(tokenizer, [toolCall("get_weather", ARG_SENTINEL_A)]);
if (error) {
return result(
"dict_arguments",
"fail",
"The JavaScript template rejected the documented dictionary argument format.",
{ error },
);
}
const finding = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A);
if (!finding.survived) {
return result(
"dict_arguments",
"fail",
"The template rendered but dropped function name or argument data.",
finding,
);
}
return result(
"dict_arguments",
"pass",
"The documented dictionary argument format renders and remains visible.",
finding,
);
}
function checkStringArguments(tokenizer) {
const prefix = [{ role: "user", content: NEUTRAL_PROMPT }];
const dictRendered = safeRender(tokenizer, [...prefix, assistantMessage()]);
const stringRendered = safeRender(tokenizer, [
...prefix,
assistantMessage({ calls: [toolCall("get_weather", ARG_SENTINEL_A, true)] }),
]);
if (dictRendered.error) {
return result("string_arguments", "skip", "Skipped because the canonical dictionary case fails.", { dict_error: dictRendered.error });
}
if (stringRendered.error) {
return result("string_arguments", "pass", "Out-of-contract string arguments are rejected loudly.", { string_error: stringRendered.error });
}
if (dictRendered.value === stringRendered.value) {
return result("string_arguments", "pass", "String arguments normalize to the same output as dictionary arguments.");
}
const backslash = countOf(stringRendered.value, '\\"') > countOf(dictRendered.value, '\\"');
const html = countOf(stringRendered.value, "&quot;") > countOf(dictRendered.value, "&quot;");
const evidence = { ...diffEvidence(dictRendered.value, stringRendered.value), extra_backslash_escaping: backslash, extra_html_escaping: html };
if (backslash || html) {
return result("string_arguments", "warn", "String arguments introduce additional escaping and may be silently miscompiled.", evidence);
}
return result("string_arguments", "pass", "Renderings differ, but no extra escaping signature was detected.", evidence);
}
function countOf(text, needle) {
return text.split(needle).length - 1;
}
function checkContentEquivalence(tokenizer) {
const prefix = [{ role: "user", content: NEUTRAL_PROMPT }];
const noneRendered = safeRender(tokenizer, [...prefix, assistantMessage({ content: null })]);
const missingRendered = safeRender(tokenizer, [...prefix, assistantMessage({ includeContent: false })]);
if (noneRendered.error || missingRendered.error) {
return result(
"content_none_equivalence",
"fail",
"content=null and a missing content key are not both renderable in JavaScript.",
{ none_error: noneRendered.error, missing_error: missingRendered.error },
);
}
if (noneRendered.value !== missingRendered.value) {
return result(
"content_none_equivalence",
"fail",
"content=null and a missing content key produce different output.",
diffEvidence(noneRendered.value, missingRendered.value),
);
}
return result("content_none_equivalence", "pass", "content=null and a missing content key are equivalent.", {});
}
function checkEmptyContent(tokenizer) {
const { probe, control, error } = differential(tokenizer, [toolCall("get_weather", ARG_SENTINEL_A)], { content: "" });
if (error) return result("empty_content", "fail", "An empty assistant content string breaks rendering.", { error });
const finding = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A);
if (!finding.survived) return result("empty_content", "fail", "The empty-content variant silently drops tool-call data.", finding);
return result("empty_content", "pass", "The empty-content variant preserves the tool call.", finding);
}
function checkParallelCalls(tokenizer) {
const calls = [
toolCall("get_weather", ARG_SENTINEL_A, false, "call_a"),
toolCall("get_time", ARG_SENTINEL_B, false, "call_b"),
];
const { probe, control, error } = differential(tokenizer, calls);
if (error) return result("parallel_calls", "fail", "The template rejected two calls in one assistant message.", { error });
const first = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A);
const second = callSurvived(probe, control, "get_time", ARG_SENTINEL_B);
const evidence = { first_call: first, second_call: second };
if (!(first.survived && second.survived)) {
return result("parallel_calls", "fail", "At least one parallel call or argument disappeared.", evidence);
}
return result("parallel_calls", "pass", "Both tool calls and their arguments remain visible.", evidence);
}
function checkToolResponse(tokenizer) {
const rendered = safeRender(tokenizer, [
{ role: "user", content: NEUTRAL_PROMPT },
assistantMessage(),
{ role: "tool", name: "get_weather", content: TOOL_RESULT_MARKER },
]);
if (rendered.error) return result("tool_response", "fail", "The template rejected a complete tool-response sequence.", { error: rendered.error });
if (!rendered.value.includes(TOOL_RESULT_MARKER)) return result("tool_response", "fail", "The template silently omitted tool-response content.", {});
return result("tool_response", "pass", "Tool-response content remains visible.", {});
}
function checkPrefixPreservation(tokenizer) {
const baseMessages = [{ role: "user", content: NEUTRAL_PROMPT }, assistantMessage()];
const extendedMessages = [...baseMessages, { role: "tool", name: "get_weather", content: TOOL_RESULT_MARKER }];
const base = safeRender(tokenizer, baseMessages);
const extended = safeRender(tokenizer, extendedMessages);
if (base.error || extended.error) {
return result("prefix_preservation", "skip", "The renderings required for this diagnostic were unavailable.", { base_error: base.error, extended_error: extended.error });
}
const preserved = extended.value.startsWith(base.value);
const evidence = { text_prefix_preserved: preserved, common_prefix_chars: commonPrefixLength(base.value, extended.value) };
if (preserved) return result("prefix_preservation", "pass", "Appending a tool response preserves the earlier rendered prefix.", evidence);
return result("prefix_preservation", "warn", "Appending a tool response rewrites earlier output; suffix extraction may be unsafe.", evidence);
}
function checkJinjaPortability(template) {
if (!template) return result("jinja_portability", "skip", "The resolved template source was unavailable.", {});
const methodMatches = [...template.matchAll(/\.(items|keys|values|lower|upper|strip|startswith|endswith)\s*\(/g)].map((match) => match[1]);
const literalMatches = [...template.matchAll(/(?:^|[^\w])(True|False|None)(?![\w])/g)].map((match) => match[1]);
const methods = [...new Set(methodMatches)].sort();
const literals = [...new Set(literalMatches)].sort();
if (methods.length || literals.length) {
return result("jinja_portability", "warn", "Python-specific constructs may differ in JavaScript or Rust Jinja engines.", { python_methods: methods, python_literals: literals });
}
return result("jinja_portability", "pass", "No common Python-specific Jinja constructs were detected.", {});
}
function prerequisiteSkip(checkId) {
return result(checkId, "skip", "Not evaluated because the canonical dictionary-argument contract failed.", { prerequisite: "dict_arguments" });
}
async function sha256(text) {
const bytes = new TextEncoder().encode(text);
const digest = await crypto.subtle.digest("SHA-256", bytes);
return [...new Uint8Array(digest)].map((value) => value.toString(16).padStart(2, "0")).join("");
}
async function lintTokenizer(modelId, tokenizer) {
let template = "";
try {
template = tokenizer.get_chat_template({ tools: TOOLS }) || "";
} catch {
template = tokenizer.chat_template || "";
}
const dictResult = checkDictArguments(tokenizer);
let checks;
if (["fail", "error"].includes(dictResult.status)) {
checks = [
dictResult,
prerequisiteSkip("string_arguments"),
prerequisiteSkip("content_none_equivalence"),
prerequisiteSkip("empty_content"),
prerequisiteSkip("parallel_calls"),
prerequisiteSkip("tool_response"),
prerequisiteSkip("prefix_preservation"),
checkJinjaPortability(template),
];
} else {
checks = [
dictResult,
checkStringArguments(tokenizer),
checkContentEquivalence(tokenizer),
checkEmptyContent(tokenizer),
checkParallelCalls(tokenizer),
checkToolResponse(tokenizer),
checkPrefixPreservation(tokenizer),
checkJinjaPortability(template),
];
}
const coverage = coverageOf(checks);
return {
model_id: modelId,
template_name: null,
coverage,
summary_line: summaryLineOf(coverage),
has_failures: tierFailures(checks, "normative"),
has_proposed_failures: tierFailures(checks, "proposed"),
metadata: {
contract_version: CONTRACT_VERSION,
engine: "@huggingface/transformers@4.2.0",
loads_model_weights: false,
template_sha256: template ? await sha256(template) : null,
runtime: navigator.userAgent,
},
checks,
};
}
// Per-tier contract coverage. This replaced the weighted 0-100 score: its weights were
// never calibrated against downstream impact, and readers used the column as a ranking.
function coverageOf(checks) {
const coverage = {};
for (const tier of ["normative", "proposed", "diagnostic"]) {
const tierChecks = checks.filter((check) => check.tier === tier);
const counts = {
passed: tierChecks.filter((c) => c.status === "pass").length,
warned: tierChecks.filter((c) => c.status === "warn").length,
failed: tierChecks.filter((c) => ["fail", "error"].includes(c.status)).length,
skipped: tierChecks.filter((c) => c.status === "skip").length,
};
counts.evaluated = tierChecks.length - counts.skipped;
coverage[tier] = counts;
}
return coverage;
}
function summaryLineOf(coverage) {
const parts = [
["required", coverage.normative],
["proposed", coverage.proposed],
].map(([label, counts]) => {
let part = `${label} ${counts.passed}/${counts.evaluated}`;
if (counts.skipped) part += ` (${counts.skipped} not evaluated)`;
return part;
});
const warnings = coverage.diagnostic.warned;
parts.push(`${warnings} diagnostic warning${warnings === 1 ? "" : "s"}`);
return parts.join(" · ");
}
function tierFailures(checks, tier) {
return checks.some((check) => check.tier === tier && ["fail", "error"].includes(check.status));
}
// A proposed-contract failure is reported on its own axis and never marks a template
// non-conforming; mirrors _row_status in src/toolcall_lint/sweep.py.
function rowStatus(report) {
if (report.has_failures) return "contract_fail";
if (report.has_proposed_failures) return "proposed_fail";
return "contract_pass";
}
function setRunProgress(label, percent, detail = "Only tokenizer and template files are requested.") {
runStateLabel.textContent = label;
const bounded = Math.max(2, Math.min(100, Number.isFinite(percent) ? percent : 8));
progressBar.style.width = `${bounded}%`;
runStatePercent.textContent = `${Math.round(bounded)}%`;
runStateDetail.textContent = detail;
}
async function runInspector(modelId) {
runButton.disabled = true;
runButton.textContent = "Inspecting…";
runState.hidden = false;
liveError.hidden = true;
liveResults.hidden = true;
setRunProgress("Loading tokenizer assets…", 4);
try {
const tokenizer = await AutoTokenizer.from_pretrained(modelId, {
progress_callback: (progress) => {
const percent = Number(progress?.progress ?? 0);
const label = progress?.file ? `Loading ${progress.file}` : "Loading tokenizer assets…";
setRunProgress(label, percent || 8);
},
});
setRunProgress("Running JavaScript contract suite…", 94, "Rendering canonical tool-call conversations locally.");
const report = await lintTokenizer(modelId, tokenizer);
lastLiveReport = report;
setRunProgress("Analysis complete", 100, "No model weights were loaded.");
renderLiveReport(report);
liveResults.hidden = false;
} catch (error) {
const message = `${error?.name || "Error"}: ${error?.message || String(error)}`;
liveError.textContent = `Unable to inspect ${modelId}.\n\n${message}\n\nThe repository may be gated, missing tokenizer assets, or incompatible with Transformers.js 4.2.0.`;
liveError.hidden = false;
setRunProgress("Inspection stopped", 100, "No result was recorded.");
} finally {
runButton.disabled = false;
runButton.textContent = "Run contract suite";
}
}
function badge(status, label = status.replaceAll("_", " ")) {
const element = document.createElement("span");
element.className = `badge badge-${status}`;
element.textContent = label;
return element;
}
function renderLiveReport(report) {
liveResults.replaceChildren();
const hero = document.createElement("div");
hero.className = "result-hero";
const title = document.createElement("div");
title.className = "result-title";
title.append(badge(rowStatus(report)));
const heading = document.createElement("h3");
heading.textContent = report.model_id;
const subtitle = document.createElement("p");
subtitle.textContent = `${report.metadata.engine} · template ${report.metadata.template_sha256?.slice(0, 12) || "unavailable"}`;
title.append(heading, subtitle);
const coverage = document.createElement("div");
coverage.className = "coverage-block";
const required = document.createElement("strong");
required.textContent = `${report.coverage.normative.passed}/${report.coverage.normative.evaluated}`;
const requiredLabel = document.createElement("span");
requiredLabel.textContent = "required contracts";
const detail = document.createElement("span");
detail.className = "coverage-detail";
detail.textContent = report.summary_line;
coverage.append(required, requiredLabel, detail);
const actions = document.createElement("div");
actions.className = "result-actions";
const copyButton = document.createElement("button");
copyButton.className = "small-button";
copyButton.textContent = "Copy JSON";
copyButton.addEventListener("click", async () => {
await navigator.clipboard.writeText(JSON.stringify(lastLiveReport, null, 2));
copyButton.textContent = "Copied";
setTimeout(() => { copyButton.textContent = "Copy JSON"; }, 1500);
});
const downloadButton = document.createElement("button");
downloadButton.className = "small-button";
downloadButton.textContent = "Download";
downloadButton.addEventListener("click", () => downloadReport(report));
actions.append(copyButton, downloadButton);
hero.append(title, coverage, actions);
const grid = document.createElement("div");
grid.className = "check-grid";
report.checks.forEach((check) => grid.append(createCheckCard(check)));
liveResults.append(hero);
const comparison = createEngineComparison(report);
if (comparison) liveResults.append(comparison);
liveResults.append(grid);
}
function createEngineComparison(report) {
const baseline = baselineDataset?.models.find((row) => row.model_id === report.model_id);
if (!baseline?.report) return null;
const pythonChecks = Object.fromEntries(
baseline.report.checks.map((check) => [check.check_id, check.status]),
);
const differences = report.checks
.filter((check) => pythonChecks[check.check_id] && pythonChecks[check.check_id] !== check.status)
.map((check) => ({
checkId: check.check_id,
python: pythonChecks[check.check_id],
javascript: check.status,
}));
const sameTemplate = baseline.template_sha256 === report.metadata.template_sha256;
const panel = document.createElement("section");
panel.className = `engine-comparison ${differences.length ? "engine-diverged" : "engine-agreed"}`;
const heading = document.createElement("div");
heading.className = "engine-comparison-heading";
const copy = document.createElement("div");
const kicker = document.createElement("span");
kicker.textContent = differences.length ? "Runtime drift detected" : "Runtime agreement";
const title = document.createElement("h3");
title.textContent = differences.length
? `${differences.length} check${differences.length === 1 ? "" : "s"} differ across engines`
: "Python and JavaScript agree on every check";
const description = document.createElement("p");
const pythonVersion = baselineDataset.environment?.transformers || "reference";
description.textContent = sameTemplate
? `Same template SHA-256, evaluated with Python Transformers ${pythonVersion} and Transformers.js 4.2.0.`
: `Template digests differ; results may combine runtime and template-revision drift.`;
copy.append(kicker, title, description);
heading.append(copy, badge(differences.length ? "warn" : "pass", sameTemplate ? "same template" : "different template"));
panel.append(heading);
if (differences.length) {
const list = document.createElement("div");
list.className = "engine-diff-list";
differences.forEach((difference) => {
const row = document.createElement("div");
const name = document.createElement("strong");
name.textContent = CHECK_TITLES[difference.checkId];
const states = document.createElement("div");
states.className = "engine-states";
const pythonLabel = document.createElement("span");
pythonLabel.textContent = `Python: ${difference.python}`;
pythonLabel.className = `check-symbol ${difference.python}`;
const arrow = document.createElement("span");
arrow.textContent = "→";
const jsLabel = document.createElement("span");
jsLabel.textContent = `JS: ${difference.javascript}`;
jsLabel.className = `check-symbol ${difference.javascript}`;
states.append(pythonLabel, arrow, jsLabel);
row.append(name, states);
list.append(row);
});
panel.append(list);
}
return panel;
}
function createCheckCard(check) {
const card = document.createElement("article");
card.className = "check-card";
const top = document.createElement("div");
top.className = "check-top";
const title = document.createElement("h4");
title.textContent = check.title;
top.append(title, badge(check.status, `${STATUS_ICON[check.status]} ${check.status}`));
const summary = document.createElement("p");
summary.textContent = check.summary;
card.append(top, summary);
if (check.evidence && Object.keys(check.evidence).length) {
const details = document.createElement("details");
const detailsTitle = document.createElement("summary");
detailsTitle.textContent = "View evidence";
const pre = document.createElement("pre");
pre.textContent = JSON.stringify(check.evidence, null, 2);
details.append(detailsTitle, pre);
card.append(details);
}
return card;
}
function downloadReport(report) {
const blob = new Blob([`${JSON.stringify(report, null, 2)}\n`], { type: "application/json" });
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = `${report.model_id.replaceAll("/", "--")}-toolcall-lint.json`;
link.click();
URL.revokeObjectURL(link.href);
}
async function loadBaseline() {
try {
const response = await fetch("./data/baseline-2026-08-19.json");
if (!response.ok) throw new Error(`HTTP ${response.status}`);
baselineDataset = await response.json();
renderBaselineSummary();
renderBaselineRows();
} catch (error) {
baselineBody.innerHTML = `<tr><td colspan="8">Baseline unavailable: ${escapeText(error.message)}</td></tr>`;
}
}
function renderBaselineSummary() {
const items = [
[baselineDataset.summary.targets, "target repositories"],
[baselineDataset.summary.loaded, "templates loaded"],
[baselineDataset.summary.contract_fail, "contract failures"],
[baselineDataset.summary.load_error, "separate load errors"],
];
baselineSummary.replaceChildren();
items.forEach(([value, label]) => {
const card = document.createElement("div");
card.className = "summary-card";
const strong = document.createElement("strong");
strong.textContent = value;
const span = document.createElement("span");
span.textContent = label;
card.append(strong, span);
baselineSummary.append(card);
});
}
function renderBaselineRows() {
if (!baselineDataset) return;
const query = modelSearch.value.trim().toLowerCase();
const filter = statusFilter.value;
const rows = baselineDataset.models.filter((row) => {
const matchesQuery = `${row.label} ${row.model_id}`.toLowerCase().includes(query);
return matchesQuery && (filter === "all" || row.status === filter);
});
baselineBody.replaceChildren();
rows.forEach((row) => baselineBody.append(createBaselineRow(row)));
tableEmpty.hidden = rows.length > 0;
}
function createBaselineRow(row) {
const tr = document.createElement("tr");
tr.tabIndex = 0;
tr.setAttribute("role", "button");
tr.setAttribute("aria-label", `View evidence for ${row.label}`);
const checks = Object.fromEntries((row.report?.checks || []).map((check) => [check.check_id, check.status]));
tr.append(
modelCell(row),
simpleCell(row.status.replaceAll("_", " "), `status-cell badge-cell`, row.status),
simpleCell(ratioOf(row.coverage?.normative), "coverage-cell"),
simpleCell(ratioOf(row.coverage?.proposed), "coverage-cell"),
checkCell(checks.dict_arguments),
checkCell(checks.parallel_calls),
checkCell(checks.tool_response),
checkCell(checks.content_none_equivalence),
checkCell(checks.prefix_preservation),
);
const open = () => openBaselineDialog(row);
tr.addEventListener("click", open);
tr.addEventListener("keydown", (event) => {
if (event.key === "Enter" || event.key === " ") {
event.preventDefault();
open();
}
});
return tr;
}
function ratioOf(counts) {
if (!counts || counts.evaluated === 0) return "—";
return `${counts.passed}/${counts.evaluated}`;
}
function modelCell(row) {
const td = document.createElement("td");
td.className = "model-cell";
const strong = document.createElement("strong");
strong.textContent = row.label;
const span = document.createElement("span");
span.textContent = row.model_id;
td.append(strong, span);
return td;
}
function simpleCell(value, className = "", status = null) {
const td = document.createElement("td");
td.className = className;
if (status) td.append(badge(status, value));
else td.textContent = value;
return td;
}
function checkCell(status) {
const td = document.createElement("td");
const span = document.createElement("span");
const normalized = status || "skip";
span.className = `check-symbol ${normalized}`;
span.textContent = `${STATUS_ICON[normalized]} ${normalized}`;
td.append(span);
return td;
}
function openBaselineDialog(row) {
dialogTitle.textContent = row.label;
dialogContent.replaceChildren();
const meta = document.createElement("div");
meta.className = "dialog-meta";
const metadata = [
["Repository", row.model_id],
["Result", row.status.replaceAll("_", " ")],
["Coverage", row.summary_line ?? "not evaluated"],
["Resolved revision", row.resolved_revision || "unavailable"],
["Template SHA-256", row.template_sha256 || "unavailable"],
["Elapsed", `${row.elapsed_seconds}s`],
];
metadata.forEach(([key, value]) => {
const item = document.createElement("div");
item.className = "meta-item";
const label = document.createElement("span");
label.textContent = key;
const strong = document.createElement("strong");
strong.textContent = value;
item.append(label, strong);
meta.append(item);
});
dialogContent.append(meta);
if (row.report) {
row.report.checks.forEach((check) => {
const article = document.createElement("article");
article.className = "dialog-check";
const head = document.createElement("div");
head.className = "dialog-check-head";
const title = document.createElement("h3");
title.textContent = check.title;
head.append(title, badge(check.status, `${STATUS_ICON[check.status]} ${check.status}`));
const summary = document.createElement("p");
summary.textContent = check.summary;
article.append(head, summary);
if (check.evidence && Object.keys(check.evidence).length) {
const pre = document.createElement("pre");
pre.className = "evidence-pre";
pre.textContent = JSON.stringify(check.evidence, null, 2);
article.append(pre);
}
dialogContent.append(article);
});
} else {
const pre = document.createElement("pre");
pre.className = "evidence-pre";
pre.textContent = JSON.stringify(row.error, null, 2);
dialogContent.append(pre);
}
dialog.showModal();
}
function escapeText(value) {
const span = document.createElement("span");
span.textContent = value;
return span.innerHTML;
}
form.addEventListener("submit", (event) => {
event.preventDefault();
const modelId = modelInput.value.trim();
if (modelId) runInspector(modelId);
});
document.querySelectorAll(".example-chip").forEach((button) => {
button.addEventListener("click", () => {
modelInput.value = button.dataset.model;
modelInput.focus();
});
});
modelSearch.addEventListener("input", renderBaselineRows);
statusFilter.addEventListener("change", renderBaselineRows);
document.querySelector("#dialog-close").addEventListener("click", () => dialog.close());
dialog.addEventListener("click", (event) => {
if (event.target === dialog) dialog.close();
});
loadBaseline();