Spaces:
Running
Running
Download web-app.js from meryemsakin/toolcall-lint: direct link, hf CLI and curl.
- Browser
- Download file 32.8 kB
-
https://huggingface.co/spaces/meryemsakin/toolcall-lint/resolve/main/web-app.js
- Command line
-
hf download hf://spaces/meryemsakin/toolcall-lint/web-app.js
-
curl -L -o web-app.js https://huggingface.co/spaces/meryemsakin/toolcall-lint/resolve/main/web-app.js
32.8 kB
| import { | |
| AutoTokenizer, | |
| env, | |
| } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0"; | |
| env.allowLocalModels = false; | |
| env.useBrowserCache = true; | |
| const CONTRACT_VERSION = "2026-08-mvp2-js1"; | |
| const TOOL_RESULT_MARKER = "TOOL_RESULT_7f3a"; | |
| // Argument values must be attributable to the assistant tool call alone. They are | |
| // deliberately absent from the tool schema below and from every prompt string, so | |
| // finding one in a rendering proves the tool call itself survived. | |
| const ARG_SENTINEL_A = "TOOLCALL_ARG_A_81d20"; | |
| const ARG_SENTINEL_B = "TOOLCALL_ARG_B_c93af"; | |
| // Prompts must not echo any string a check searches for. | |
| const NEUTRAL_PROMPT = "Use the available tools."; | |
| // Authoritative tier for every check; mirrors CHECK_TIERS in src/toolcall_lint/checks.py. | |
| // `content_none_equivalence` is proposed rather than required because the maintainer | |
| // clarification it encodes (transformers PR #45422) has not merged. | |
| const CHECK_TIERS = { | |
| dict_arguments: "normative", | |
| empty_content: "normative", | |
| parallel_calls: "normative", | |
| tool_response: "normative", | |
| content_none_equivalence: "proposed", | |
| string_arguments: "diagnostic", | |
| prefix_preservation: "diagnostic", | |
| jinja_portability: "diagnostic", | |
| }; | |
| const STATUS_ICON = { pass: "✓", warn: "!", fail: "×", skip: "–", error: "×" }; | |
| const CHECK_TITLES = { | |
| dict_arguments: "Dictionary tool arguments", | |
| string_arguments: "String argument handling", | |
| content_none_equivalence: "Missing content equivalence", | |
| empty_content: "Empty assistant content", | |
| parallel_calls: "Parallel tool calls", | |
| tool_response: "Tool response preservation", | |
| prefix_preservation: "Training prefix preservation", | |
| jinja_portability: "Cross-engine Jinja portability", | |
| }; | |
| const TOOLS = [ | |
| { | |
| type: "function", | |
| function: { | |
| name: "get_weather", | |
| description: "Get the weather for a city.", | |
| parameters: { | |
| type: "object", | |
| properties: { city: { type: "string" } }, | |
| required: ["city"], | |
| additionalProperties: false, | |
| }, | |
| }, | |
| }, | |
| { | |
| type: "function", | |
| function: { | |
| name: "get_time", | |
| description: "Get the local time for a city.", | |
| parameters: { | |
| type: "object", | |
| properties: { city: { type: "string" } }, | |
| required: ["city"], | |
| additionalProperties: false, | |
| }, | |
| }, | |
| }, | |
| ]; | |
| const form = document.querySelector("#inspect-form"); | |
| const modelInput = document.querySelector("#model-id"); | |
| const runButton = document.querySelector("#run-button"); | |
| const runState = document.querySelector("#run-state"); | |
| const runStateLabel = document.querySelector("#run-state-label"); | |
| const runStatePercent = document.querySelector("#run-state-percent"); | |
| const runStateDetail = document.querySelector("#run-state-detail"); | |
| const progressBar = document.querySelector("#progress-bar"); | |
| const liveError = document.querySelector("#live-error"); | |
| const liveResults = document.querySelector("#live-results"); | |
| const baselineBody = document.querySelector("#baseline-body"); | |
| const baselineSummary = document.querySelector("#baseline-summary"); | |
| const modelSearch = document.querySelector("#model-search"); | |
| const statusFilter = document.querySelector("#status-filter"); | |
| const tableEmpty = document.querySelector("#table-empty"); | |
| const dialog = document.querySelector("#detail-dialog"); | |
| const dialogTitle = document.querySelector("#dialog-title"); | |
| const dialogContent = document.querySelector("#dialog-content"); | |
| let baselineDataset = null; | |
| let lastLiveReport = null; | |
| function toolCall(name, city, argumentsAsString = false, callId = "call_a") { | |
| const args = argumentsAsString ? JSON.stringify({ city }) : { city }; | |
| return { | |
| id: callId, | |
| type: "function", | |
| function: { name, arguments: args }, | |
| }; | |
| } | |
| function assistantMessage({ content = "", includeContent = true, calls = null } = {}) { | |
| const message = { | |
| role: "assistant", | |
| tool_calls: calls || [toolCall("get_weather", ARG_SENTINEL_A)], | |
| }; | |
| if (includeContent) message.content = content; | |
| return message; | |
| } | |
| function safeRender(tokenizer, messages) { | |
| try { | |
| const value = tokenizer.apply_chat_template(messages, { | |
| tools: TOOLS, | |
| tokenize: false, | |
| add_generation_prompt: false, | |
| }); | |
| return { value, error: null }; | |
| } catch (error) { | |
| return { value: null, error: `${error?.name || "Error"}: ${error?.message || String(error)}` }; | |
| } | |
| } | |
| function result(checkId, status, summary, evidence = {}) { | |
| return { | |
| check_id: checkId, | |
| title: CHECK_TITLES[checkId], | |
| status, | |
| summary, | |
| evidence, | |
| tier: CHECK_TIERS[checkId], | |
| }; | |
| } | |
| // Render the same conversation with and without `calls`. The control rendering | |
| // isolates the output attributable to the assistant tool call, so a template that | |
| // echoes tool schemas or prompt text cannot satisfy a check by accident. | |
| function differential(tokenizer, calls, { content = "", includeContent = true } = {}) { | |
| const userTurn = { role: "user", content: NEUTRAL_PROMPT }; | |
| const probe = safeRender(tokenizer, [userTurn, assistantMessage({ content, includeContent, calls })]); | |
| if (probe.error) return { probe: null, control: null, error: probe.error }; | |
| const controlMessage = { role: "assistant" }; | |
| if (includeContent) controlMessage.content = content; | |
| const control = safeRender(tokenizer, [userTurn, controlMessage]); | |
| return { probe: probe.value, control: control.error ? null : control.value, error: null }; | |
| } | |
| // The argument value is a sentinel that exists nowhere else in the input, so its | |
| // presence is direct evidence. The function name also appears in the tool schema, | |
| // so it only counts when the tool call adds an occurrence the control lacks. | |
| function callSurvived(probe, control, name, sentinel) { | |
| const finding = { | |
| argument_sentinel_present: probe.includes(sentinel), | |
| name_occurrences_with_call: countOf(probe, name), | |
| }; | |
| if (control === null) { | |
| finding.differential = "unavailable"; | |
| finding.survived = finding.argument_sentinel_present; | |
| return finding; | |
| } | |
| finding.name_occurrences_without_call = countOf(control, name); | |
| finding.name_attributable_to_call = countOf(probe, name) > countOf(control, name); | |
| finding.argument_sentinel_leaked_into_control = control.includes(sentinel); | |
| finding.survived = | |
| finding.argument_sentinel_present && | |
| finding.name_attributable_to_call && | |
| !finding.argument_sentinel_leaked_into_control; | |
| return finding; | |
| } | |
| function commonPrefixLength(left, right) { | |
| const limit = Math.min(left.length, right.length); | |
| for (let index = 0; index < limit; index += 1) { | |
| if (left[index] !== right[index]) return index; | |
| } | |
| return limit; | |
| } | |
| function diffEvidence(left, right, radius = 100) { | |
| const index = commonPrefixLength(left, right); | |
| const start = Math.max(0, index - radius); | |
| const end = index + radius; | |
| return { | |
| first_difference: index, | |
| canonical_window: left.slice(start, end), | |
| comparison_window: right.slice(start, end), | |
| }; | |
| } | |
| function checkDictArguments(tokenizer) { | |
| const { probe, control, error } = differential(tokenizer, [toolCall("get_weather", ARG_SENTINEL_A)]); | |
| if (error) { | |
| return result( | |
| "dict_arguments", | |
| "fail", | |
| "The JavaScript template rejected the documented dictionary argument format.", | |
| { error }, | |
| ); | |
| } | |
| const finding = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A); | |
| if (!finding.survived) { | |
| return result( | |
| "dict_arguments", | |
| "fail", | |
| "The template rendered but dropped function name or argument data.", | |
| finding, | |
| ); | |
| } | |
| return result( | |
| "dict_arguments", | |
| "pass", | |
| "The documented dictionary argument format renders and remains visible.", | |
| finding, | |
| ); | |
| } | |
| function checkStringArguments(tokenizer) { | |
| const prefix = [{ role: "user", content: NEUTRAL_PROMPT }]; | |
| const dictRendered = safeRender(tokenizer, [...prefix, assistantMessage()]); | |
| const stringRendered = safeRender(tokenizer, [ | |
| ...prefix, | |
| assistantMessage({ calls: [toolCall("get_weather", ARG_SENTINEL_A, true)] }), | |
| ]); | |
| if (dictRendered.error) { | |
| return result("string_arguments", "skip", "Skipped because the canonical dictionary case fails.", { dict_error: dictRendered.error }); | |
| } | |
| if (stringRendered.error) { | |
| return result("string_arguments", "pass", "Out-of-contract string arguments are rejected loudly.", { string_error: stringRendered.error }); | |
| } | |
| if (dictRendered.value === stringRendered.value) { | |
| return result("string_arguments", "pass", "String arguments normalize to the same output as dictionary arguments."); | |
| } | |
| const backslash = countOf(stringRendered.value, '\\"') > countOf(dictRendered.value, '\\"'); | |
| const html = countOf(stringRendered.value, """) > countOf(dictRendered.value, """); | |
| const evidence = { ...diffEvidence(dictRendered.value, stringRendered.value), extra_backslash_escaping: backslash, extra_html_escaping: html }; | |
| if (backslash || html) { | |
| return result("string_arguments", "warn", "String arguments introduce additional escaping and may be silently miscompiled.", evidence); | |
| } | |
| return result("string_arguments", "pass", "Renderings differ, but no extra escaping signature was detected.", evidence); | |
| } | |
| function countOf(text, needle) { | |
| return text.split(needle).length - 1; | |
| } | |
| function checkContentEquivalence(tokenizer) { | |
| const prefix = [{ role: "user", content: NEUTRAL_PROMPT }]; | |
| const noneRendered = safeRender(tokenizer, [...prefix, assistantMessage({ content: null })]); | |
| const missingRendered = safeRender(tokenizer, [...prefix, assistantMessage({ includeContent: false })]); | |
| if (noneRendered.error || missingRendered.error) { | |
| return result( | |
| "content_none_equivalence", | |
| "fail", | |
| "content=null and a missing content key are not both renderable in JavaScript.", | |
| { none_error: noneRendered.error, missing_error: missingRendered.error }, | |
| ); | |
| } | |
| if (noneRendered.value !== missingRendered.value) { | |
| return result( | |
| "content_none_equivalence", | |
| "fail", | |
| "content=null and a missing content key produce different output.", | |
| diffEvidence(noneRendered.value, missingRendered.value), | |
| ); | |
| } | |
| return result("content_none_equivalence", "pass", "content=null and a missing content key are equivalent.", {}); | |
| } | |
| function checkEmptyContent(tokenizer) { | |
| const { probe, control, error } = differential(tokenizer, [toolCall("get_weather", ARG_SENTINEL_A)], { content: "" }); | |
| if (error) return result("empty_content", "fail", "An empty assistant content string breaks rendering.", { error }); | |
| const finding = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A); | |
| if (!finding.survived) return result("empty_content", "fail", "The empty-content variant silently drops tool-call data.", finding); | |
| return result("empty_content", "pass", "The empty-content variant preserves the tool call.", finding); | |
| } | |
| function checkParallelCalls(tokenizer) { | |
| const calls = [ | |
| toolCall("get_weather", ARG_SENTINEL_A, false, "call_a"), | |
| toolCall("get_time", ARG_SENTINEL_B, false, "call_b"), | |
| ]; | |
| const { probe, control, error } = differential(tokenizer, calls); | |
| if (error) return result("parallel_calls", "fail", "The template rejected two calls in one assistant message.", { error }); | |
| const first = callSurvived(probe, control, "get_weather", ARG_SENTINEL_A); | |
| const second = callSurvived(probe, control, "get_time", ARG_SENTINEL_B); | |
| const evidence = { first_call: first, second_call: second }; | |
| if (!(first.survived && second.survived)) { | |
| return result("parallel_calls", "fail", "At least one parallel call or argument disappeared.", evidence); | |
| } | |
| return result("parallel_calls", "pass", "Both tool calls and their arguments remain visible.", evidence); | |
| } | |
| function checkToolResponse(tokenizer) { | |
| const rendered = safeRender(tokenizer, [ | |
| { role: "user", content: NEUTRAL_PROMPT }, | |
| assistantMessage(), | |
| { role: "tool", name: "get_weather", content: TOOL_RESULT_MARKER }, | |
| ]); | |
| if (rendered.error) return result("tool_response", "fail", "The template rejected a complete tool-response sequence.", { error: rendered.error }); | |
| if (!rendered.value.includes(TOOL_RESULT_MARKER)) return result("tool_response", "fail", "The template silently omitted tool-response content.", {}); | |
| return result("tool_response", "pass", "Tool-response content remains visible.", {}); | |
| } | |
| function checkPrefixPreservation(tokenizer) { | |
| const baseMessages = [{ role: "user", content: NEUTRAL_PROMPT }, assistantMessage()]; | |
| const extendedMessages = [...baseMessages, { role: "tool", name: "get_weather", content: TOOL_RESULT_MARKER }]; | |
| const base = safeRender(tokenizer, baseMessages); | |
| const extended = safeRender(tokenizer, extendedMessages); | |
| if (base.error || extended.error) { | |
| return result("prefix_preservation", "skip", "The renderings required for this diagnostic were unavailable.", { base_error: base.error, extended_error: extended.error }); | |
| } | |
| const preserved = extended.value.startsWith(base.value); | |
| const evidence = { text_prefix_preserved: preserved, common_prefix_chars: commonPrefixLength(base.value, extended.value) }; | |
| if (preserved) return result("prefix_preservation", "pass", "Appending a tool response preserves the earlier rendered prefix.", evidence); | |
| return result("prefix_preservation", "warn", "Appending a tool response rewrites earlier output; suffix extraction may be unsafe.", evidence); | |
| } | |
| function checkJinjaPortability(template) { | |
| if (!template) return result("jinja_portability", "skip", "The resolved template source was unavailable.", {}); | |
| const methodMatches = [...template.matchAll(/\.(items|keys|values|lower|upper|strip|startswith|endswith)\s*\(/g)].map((match) => match[1]); | |
| const literalMatches = [...template.matchAll(/(?:^|[^\w])(True|False|None)(?![\w])/g)].map((match) => match[1]); | |
| const methods = [...new Set(methodMatches)].sort(); | |
| const literals = [...new Set(literalMatches)].sort(); | |
| if (methods.length || literals.length) { | |
| return result("jinja_portability", "warn", "Python-specific constructs may differ in JavaScript or Rust Jinja engines.", { python_methods: methods, python_literals: literals }); | |
| } | |
| return result("jinja_portability", "pass", "No common Python-specific Jinja constructs were detected.", {}); | |
| } | |
| function prerequisiteSkip(checkId) { | |
| return result(checkId, "skip", "Not evaluated because the canonical dictionary-argument contract failed.", { prerequisite: "dict_arguments" }); | |
| } | |
| async function sha256(text) { | |
| const bytes = new TextEncoder().encode(text); | |
| const digest = await crypto.subtle.digest("SHA-256", bytes); | |
| return [...new Uint8Array(digest)].map((value) => value.toString(16).padStart(2, "0")).join(""); | |
| } | |
| async function lintTokenizer(modelId, tokenizer) { | |
| let template = ""; | |
| try { | |
| template = tokenizer.get_chat_template({ tools: TOOLS }) || ""; | |
| } catch { | |
| template = tokenizer.chat_template || ""; | |
| } | |
| const dictResult = checkDictArguments(tokenizer); | |
| let checks; | |
| if (["fail", "error"].includes(dictResult.status)) { | |
| checks = [ | |
| dictResult, | |
| prerequisiteSkip("string_arguments"), | |
| prerequisiteSkip("content_none_equivalence"), | |
| prerequisiteSkip("empty_content"), | |
| prerequisiteSkip("parallel_calls"), | |
| prerequisiteSkip("tool_response"), | |
| prerequisiteSkip("prefix_preservation"), | |
| checkJinjaPortability(template), | |
| ]; | |
| } else { | |
| checks = [ | |
| dictResult, | |
| checkStringArguments(tokenizer), | |
| checkContentEquivalence(tokenizer), | |
| checkEmptyContent(tokenizer), | |
| checkParallelCalls(tokenizer), | |
| checkToolResponse(tokenizer), | |
| checkPrefixPreservation(tokenizer), | |
| checkJinjaPortability(template), | |
| ]; | |
| } | |
| const coverage = coverageOf(checks); | |
| return { | |
| model_id: modelId, | |
| template_name: null, | |
| coverage, | |
| summary_line: summaryLineOf(coverage), | |
| has_failures: tierFailures(checks, "normative"), | |
| has_proposed_failures: tierFailures(checks, "proposed"), | |
| metadata: { | |
| contract_version: CONTRACT_VERSION, | |
| engine: "@huggingface/transformers@4.2.0", | |
| loads_model_weights: false, | |
| template_sha256: template ? await sha256(template) : null, | |
| runtime: navigator.userAgent, | |
| }, | |
| checks, | |
| }; | |
| } | |
| // Per-tier contract coverage. This replaced the weighted 0-100 score: its weights were | |
| // never calibrated against downstream impact, and readers used the column as a ranking. | |
| function coverageOf(checks) { | |
| const coverage = {}; | |
| for (const tier of ["normative", "proposed", "diagnostic"]) { | |
| const tierChecks = checks.filter((check) => check.tier === tier); | |
| const counts = { | |
| passed: tierChecks.filter((c) => c.status === "pass").length, | |
| warned: tierChecks.filter((c) => c.status === "warn").length, | |
| failed: tierChecks.filter((c) => ["fail", "error"].includes(c.status)).length, | |
| skipped: tierChecks.filter((c) => c.status === "skip").length, | |
| }; | |
| counts.evaluated = tierChecks.length - counts.skipped; | |
| coverage[tier] = counts; | |
| } | |
| return coverage; | |
| } | |
| function summaryLineOf(coverage) { | |
| const parts = [ | |
| ["required", coverage.normative], | |
| ["proposed", coverage.proposed], | |
| ].map(([label, counts]) => { | |
| let part = `${label} ${counts.passed}/${counts.evaluated}`; | |
| if (counts.skipped) part += ` (${counts.skipped} not evaluated)`; | |
| return part; | |
| }); | |
| const warnings = coverage.diagnostic.warned; | |
| parts.push(`${warnings} diagnostic warning${warnings === 1 ? "" : "s"}`); | |
| return parts.join(" · "); | |
| } | |
| function tierFailures(checks, tier) { | |
| return checks.some((check) => check.tier === tier && ["fail", "error"].includes(check.status)); | |
| } | |
| // A proposed-contract failure is reported on its own axis and never marks a template | |
| // non-conforming; mirrors _row_status in src/toolcall_lint/sweep.py. | |
| function rowStatus(report) { | |
| if (report.has_failures) return "contract_fail"; | |
| if (report.has_proposed_failures) return "proposed_fail"; | |
| return "contract_pass"; | |
| } | |
| function setRunProgress(label, percent, detail = "Only tokenizer and template files are requested.") { | |
| runStateLabel.textContent = label; | |
| const bounded = Math.max(2, Math.min(100, Number.isFinite(percent) ? percent : 8)); | |
| progressBar.style.width = `${bounded}%`; | |
| runStatePercent.textContent = `${Math.round(bounded)}%`; | |
| runStateDetail.textContent = detail; | |
| } | |
| async function runInspector(modelId) { | |
| runButton.disabled = true; | |
| runButton.textContent = "Inspecting…"; | |
| runState.hidden = false; | |
| liveError.hidden = true; | |
| liveResults.hidden = true; | |
| setRunProgress("Loading tokenizer assets…", 4); | |
| try { | |
| const tokenizer = await AutoTokenizer.from_pretrained(modelId, { | |
| progress_callback: (progress) => { | |
| const percent = Number(progress?.progress ?? 0); | |
| const label = progress?.file ? `Loading ${progress.file}` : "Loading tokenizer assets…"; | |
| setRunProgress(label, percent || 8); | |
| }, | |
| }); | |
| setRunProgress("Running JavaScript contract suite…", 94, "Rendering canonical tool-call conversations locally."); | |
| const report = await lintTokenizer(modelId, tokenizer); | |
| lastLiveReport = report; | |
| setRunProgress("Analysis complete", 100, "No model weights were loaded."); | |
| renderLiveReport(report); | |
| liveResults.hidden = false; | |
| } catch (error) { | |
| const message = `${error?.name || "Error"}: ${error?.message || String(error)}`; | |
| liveError.textContent = `Unable to inspect ${modelId}.\n\n${message}\n\nThe repository may be gated, missing tokenizer assets, or incompatible with Transformers.js 4.2.0.`; | |
| liveError.hidden = false; | |
| setRunProgress("Inspection stopped", 100, "No result was recorded."); | |
| } finally { | |
| runButton.disabled = false; | |
| runButton.textContent = "Run contract suite"; | |
| } | |
| } | |
| function badge(status, label = status.replaceAll("_", " ")) { | |
| const element = document.createElement("span"); | |
| element.className = `badge badge-${status}`; | |
| element.textContent = label; | |
| return element; | |
| } | |
| function renderLiveReport(report) { | |
| liveResults.replaceChildren(); | |
| const hero = document.createElement("div"); | |
| hero.className = "result-hero"; | |
| const title = document.createElement("div"); | |
| title.className = "result-title"; | |
| title.append(badge(rowStatus(report))); | |
| const heading = document.createElement("h3"); | |
| heading.textContent = report.model_id; | |
| const subtitle = document.createElement("p"); | |
| subtitle.textContent = `${report.metadata.engine} · template ${report.metadata.template_sha256?.slice(0, 12) || "unavailable"}`; | |
| title.append(heading, subtitle); | |
| const coverage = document.createElement("div"); | |
| coverage.className = "coverage-block"; | |
| const required = document.createElement("strong"); | |
| required.textContent = `${report.coverage.normative.passed}/${report.coverage.normative.evaluated}`; | |
| const requiredLabel = document.createElement("span"); | |
| requiredLabel.textContent = "required contracts"; | |
| const detail = document.createElement("span"); | |
| detail.className = "coverage-detail"; | |
| detail.textContent = report.summary_line; | |
| coverage.append(required, requiredLabel, detail); | |
| const actions = document.createElement("div"); | |
| actions.className = "result-actions"; | |
| const copyButton = document.createElement("button"); | |
| copyButton.className = "small-button"; | |
| copyButton.textContent = "Copy JSON"; | |
| copyButton.addEventListener("click", async () => { | |
| await navigator.clipboard.writeText(JSON.stringify(lastLiveReport, null, 2)); | |
| copyButton.textContent = "Copied"; | |
| setTimeout(() => { copyButton.textContent = "Copy JSON"; }, 1500); | |
| }); | |
| const downloadButton = document.createElement("button"); | |
| downloadButton.className = "small-button"; | |
| downloadButton.textContent = "Download"; | |
| downloadButton.addEventListener("click", () => downloadReport(report)); | |
| actions.append(copyButton, downloadButton); | |
| hero.append(title, coverage, actions); | |
| const grid = document.createElement("div"); | |
| grid.className = "check-grid"; | |
| report.checks.forEach((check) => grid.append(createCheckCard(check))); | |
| liveResults.append(hero); | |
| const comparison = createEngineComparison(report); | |
| if (comparison) liveResults.append(comparison); | |
| liveResults.append(grid); | |
| } | |
| function createEngineComparison(report) { | |
| const baseline = baselineDataset?.models.find((row) => row.model_id === report.model_id); | |
| if (!baseline?.report) return null; | |
| const pythonChecks = Object.fromEntries( | |
| baseline.report.checks.map((check) => [check.check_id, check.status]), | |
| ); | |
| const differences = report.checks | |
| .filter((check) => pythonChecks[check.check_id] && pythonChecks[check.check_id] !== check.status) | |
| .map((check) => ({ | |
| checkId: check.check_id, | |
| python: pythonChecks[check.check_id], | |
| javascript: check.status, | |
| })); | |
| const sameTemplate = baseline.template_sha256 === report.metadata.template_sha256; | |
| const panel = document.createElement("section"); | |
| panel.className = `engine-comparison ${differences.length ? "engine-diverged" : "engine-agreed"}`; | |
| const heading = document.createElement("div"); | |
| heading.className = "engine-comparison-heading"; | |
| const copy = document.createElement("div"); | |
| const kicker = document.createElement("span"); | |
| kicker.textContent = differences.length ? "Runtime drift detected" : "Runtime agreement"; | |
| const title = document.createElement("h3"); | |
| title.textContent = differences.length | |
| ? `${differences.length} check${differences.length === 1 ? "" : "s"} differ across engines` | |
| : "Python and JavaScript agree on every check"; | |
| const description = document.createElement("p"); | |
| const pythonVersion = baselineDataset.environment?.transformers || "reference"; | |
| description.textContent = sameTemplate | |
| ? `Same template SHA-256, evaluated with Python Transformers ${pythonVersion} and Transformers.js 4.2.0.` | |
| : `Template digests differ; results may combine runtime and template-revision drift.`; | |
| copy.append(kicker, title, description); | |
| heading.append(copy, badge(differences.length ? "warn" : "pass", sameTemplate ? "same template" : "different template")); | |
| panel.append(heading); | |
| if (differences.length) { | |
| const list = document.createElement("div"); | |
| list.className = "engine-diff-list"; | |
| differences.forEach((difference) => { | |
| const row = document.createElement("div"); | |
| const name = document.createElement("strong"); | |
| name.textContent = CHECK_TITLES[difference.checkId]; | |
| const states = document.createElement("div"); | |
| states.className = "engine-states"; | |
| const pythonLabel = document.createElement("span"); | |
| pythonLabel.textContent = `Python: ${difference.python}`; | |
| pythonLabel.className = `check-symbol ${difference.python}`; | |
| const arrow = document.createElement("span"); | |
| arrow.textContent = "→"; | |
| const jsLabel = document.createElement("span"); | |
| jsLabel.textContent = `JS: ${difference.javascript}`; | |
| jsLabel.className = `check-symbol ${difference.javascript}`; | |
| states.append(pythonLabel, arrow, jsLabel); | |
| row.append(name, states); | |
| list.append(row); | |
| }); | |
| panel.append(list); | |
| } | |
| return panel; | |
| } | |
| function createCheckCard(check) { | |
| const card = document.createElement("article"); | |
| card.className = "check-card"; | |
| const top = document.createElement("div"); | |
| top.className = "check-top"; | |
| const title = document.createElement("h4"); | |
| title.textContent = check.title; | |
| top.append(title, badge(check.status, `${STATUS_ICON[check.status]} ${check.status}`)); | |
| const summary = document.createElement("p"); | |
| summary.textContent = check.summary; | |
| card.append(top, summary); | |
| if (check.evidence && Object.keys(check.evidence).length) { | |
| const details = document.createElement("details"); | |
| const detailsTitle = document.createElement("summary"); | |
| detailsTitle.textContent = "View evidence"; | |
| const pre = document.createElement("pre"); | |
| pre.textContent = JSON.stringify(check.evidence, null, 2); | |
| details.append(detailsTitle, pre); | |
| card.append(details); | |
| } | |
| return card; | |
| } | |
| function downloadReport(report) { | |
| const blob = new Blob([`${JSON.stringify(report, null, 2)}\n`], { type: "application/json" }); | |
| const link = document.createElement("a"); | |
| link.href = URL.createObjectURL(blob); | |
| link.download = `${report.model_id.replaceAll("/", "--")}-toolcall-lint.json`; | |
| link.click(); | |
| URL.revokeObjectURL(link.href); | |
| } | |
| async function loadBaseline() { | |
| try { | |
| const response = await fetch("./data/baseline-2026-08-19.json"); | |
| if (!response.ok) throw new Error(`HTTP ${response.status}`); | |
| baselineDataset = await response.json(); | |
| renderBaselineSummary(); | |
| renderBaselineRows(); | |
| } catch (error) { | |
| baselineBody.innerHTML = `<tr><td colspan="8">Baseline unavailable: ${escapeText(error.message)}</td></tr>`; | |
| } | |
| } | |
| function renderBaselineSummary() { | |
| const items = [ | |
| [baselineDataset.summary.targets, "target repositories"], | |
| [baselineDataset.summary.loaded, "templates loaded"], | |
| [baselineDataset.summary.contract_fail, "contract failures"], | |
| [baselineDataset.summary.load_error, "separate load errors"], | |
| ]; | |
| baselineSummary.replaceChildren(); | |
| items.forEach(([value, label]) => { | |
| const card = document.createElement("div"); | |
| card.className = "summary-card"; | |
| const strong = document.createElement("strong"); | |
| strong.textContent = value; | |
| const span = document.createElement("span"); | |
| span.textContent = label; | |
| card.append(strong, span); | |
| baselineSummary.append(card); | |
| }); | |
| } | |
| function renderBaselineRows() { | |
| if (!baselineDataset) return; | |
| const query = modelSearch.value.trim().toLowerCase(); | |
| const filter = statusFilter.value; | |
| const rows = baselineDataset.models.filter((row) => { | |
| const matchesQuery = `${row.label} ${row.model_id}`.toLowerCase().includes(query); | |
| return matchesQuery && (filter === "all" || row.status === filter); | |
| }); | |
| baselineBody.replaceChildren(); | |
| rows.forEach((row) => baselineBody.append(createBaselineRow(row))); | |
| tableEmpty.hidden = rows.length > 0; | |
| } | |
| function createBaselineRow(row) { | |
| const tr = document.createElement("tr"); | |
| tr.tabIndex = 0; | |
| tr.setAttribute("role", "button"); | |
| tr.setAttribute("aria-label", `View evidence for ${row.label}`); | |
| const checks = Object.fromEntries((row.report?.checks || []).map((check) => [check.check_id, check.status])); | |
| tr.append( | |
| modelCell(row), | |
| simpleCell(row.status.replaceAll("_", " "), `status-cell badge-cell`, row.status), | |
| simpleCell(ratioOf(row.coverage?.normative), "coverage-cell"), | |
| simpleCell(ratioOf(row.coverage?.proposed), "coverage-cell"), | |
| checkCell(checks.dict_arguments), | |
| checkCell(checks.parallel_calls), | |
| checkCell(checks.tool_response), | |
| checkCell(checks.content_none_equivalence), | |
| checkCell(checks.prefix_preservation), | |
| ); | |
| const open = () => openBaselineDialog(row); | |
| tr.addEventListener("click", open); | |
| tr.addEventListener("keydown", (event) => { | |
| if (event.key === "Enter" || event.key === " ") { | |
| event.preventDefault(); | |
| open(); | |
| } | |
| }); | |
| return tr; | |
| } | |
| function ratioOf(counts) { | |
| if (!counts || counts.evaluated === 0) return "—"; | |
| return `${counts.passed}/${counts.evaluated}`; | |
| } | |
| function modelCell(row) { | |
| const td = document.createElement("td"); | |
| td.className = "model-cell"; | |
| const strong = document.createElement("strong"); | |
| strong.textContent = row.label; | |
| const span = document.createElement("span"); | |
| span.textContent = row.model_id; | |
| td.append(strong, span); | |
| return td; | |
| } | |
| function simpleCell(value, className = "", status = null) { | |
| const td = document.createElement("td"); | |
| td.className = className; | |
| if (status) td.append(badge(status, value)); | |
| else td.textContent = value; | |
| return td; | |
| } | |
| function checkCell(status) { | |
| const td = document.createElement("td"); | |
| const span = document.createElement("span"); | |
| const normalized = status || "skip"; | |
| span.className = `check-symbol ${normalized}`; | |
| span.textContent = `${STATUS_ICON[normalized]} ${normalized}`; | |
| td.append(span); | |
| return td; | |
| } | |
| function openBaselineDialog(row) { | |
| dialogTitle.textContent = row.label; | |
| dialogContent.replaceChildren(); | |
| const meta = document.createElement("div"); | |
| meta.className = "dialog-meta"; | |
| const metadata = [ | |
| ["Repository", row.model_id], | |
| ["Result", row.status.replaceAll("_", " ")], | |
| ["Coverage", row.summary_line ?? "not evaluated"], | |
| ["Resolved revision", row.resolved_revision || "unavailable"], | |
| ["Template SHA-256", row.template_sha256 || "unavailable"], | |
| ["Elapsed", `${row.elapsed_seconds}s`], | |
| ]; | |
| metadata.forEach(([key, value]) => { | |
| const item = document.createElement("div"); | |
| item.className = "meta-item"; | |
| const label = document.createElement("span"); | |
| label.textContent = key; | |
| const strong = document.createElement("strong"); | |
| strong.textContent = value; | |
| item.append(label, strong); | |
| meta.append(item); | |
| }); | |
| dialogContent.append(meta); | |
| if (row.report) { | |
| row.report.checks.forEach((check) => { | |
| const article = document.createElement("article"); | |
| article.className = "dialog-check"; | |
| const head = document.createElement("div"); | |
| head.className = "dialog-check-head"; | |
| const title = document.createElement("h3"); | |
| title.textContent = check.title; | |
| head.append(title, badge(check.status, `${STATUS_ICON[check.status]} ${check.status}`)); | |
| const summary = document.createElement("p"); | |
| summary.textContent = check.summary; | |
| article.append(head, summary); | |
| if (check.evidence && Object.keys(check.evidence).length) { | |
| const pre = document.createElement("pre"); | |
| pre.className = "evidence-pre"; | |
| pre.textContent = JSON.stringify(check.evidence, null, 2); | |
| article.append(pre); | |
| } | |
| dialogContent.append(article); | |
| }); | |
| } else { | |
| const pre = document.createElement("pre"); | |
| pre.className = "evidence-pre"; | |
| pre.textContent = JSON.stringify(row.error, null, 2); | |
| dialogContent.append(pre); | |
| } | |
| dialog.showModal(); | |
| } | |
| function escapeText(value) { | |
| const span = document.createElement("span"); | |
| span.textContent = value; | |
| return span.innerHTML; | |
| } | |
| form.addEventListener("submit", (event) => { | |
| event.preventDefault(); | |
| const modelId = modelInput.value.trim(); | |
| if (modelId) runInspector(modelId); | |
| }); | |
| document.querySelectorAll(".example-chip").forEach((button) => { | |
| button.addEventListener("click", () => { | |
| modelInput.value = button.dataset.model; | |
| modelInput.focus(); | |
| }); | |
| }); | |
| modelSearch.addEventListener("input", renderBaselineRows); | |
| statusFilter.addEventListener("change", renderBaselineRows); | |
| document.querySelector("#dialog-close").addEventListener("click", () => dialog.close()); | |
| dialog.addEventListener("click", (event) => { | |
| if (event.target === dialog) dialog.close(); | |
| }); | |
| loadBaseline(); | |