import assert from "node:assert/strict"; import test from "node:test"; import { readFile } from "node:fs/promises"; // @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports. import { parseScoreExport, scoreAt, scorePath } from "../lib/model-scores.ts"; const fixture = () => ({ schema_version: 1, model: "maai", name: "MaAI", score: "1-p_now[user]", sample_interval_ms: 100, calls: [{ call_id: "demo", audio_path: "demo.mp3", audio_sha256: "a".repeat(64), duration_ms: 1000, points: [[100, .2], [200, .9], [900, .3]], checkpoints: [[150, .8]] }] }); test("model scores accept real ordered samples and independent checkpoint times", () => { assert.equal(parseScoreExport(fixture(), "maai").calls[0].checkpoints[0][0], 150); }); test("invalid exports cannot masquerade as zero scores or another model", () => { for (const points of [[[100, null]], [[100, 1.1]], [[100, -1]], [[100, .5], [100, .6]], [[200, .5], [100, .6]], [[1001, .5]]]) { const data = fixture(); data.calls[0].points = points as number[][]; assert.throws(() => parseScoreExport(data, "maai")); } assert.throws(() => parseScoreExport(fixture(), "smart_turn")); const duplicate = fixture(); duplicate.calls.push(duplicate.calls[0]); assert.throws(() => parseScoreExport(duplicate, "maai")); }); test("playhead values never use a future sample or carry through missing data", () => { const points: [number, number][] = [[100, .2], [200, .9], [900, .3]]; assert.equal(scoreAt(points, 99, 200), null); assert.deepEqual(scoreAt(points, 150, 200), [100, .2]); assert.deepEqual(scoreAt(points, 200, 200), [200, .9]); assert.equal(scoreAt(points, 600, 200), null); assert.equal(scoreAt([], 100, 200), null); }); test("curve gaps are not connected and steps remain on the audio clock", () => { const path = scorePath([[100, .2], [200, .9], [900, .3]], 1000, 500, 100, 100); assert.equal(path, "M50.00,72.80H100.00V19.60M450.00,65.20"); }); test("recorded inference exports agree on audio identity and retain all 44 checkpoints", async () => { const files = [ ["maai", "maai-ch-kyoto"], ["smart_turn", "smart-turn-v3.2"], ["silero_vad", "silero-vad"], ["livekit_mini", "livekit-v1-mini"], ["turnsense", "turnsense-1.0"], ] as const; const exports = await Promise.all(files.map(async ([model, directory]) => parseScoreExport( JSON.parse(await readFile(new URL(`../../../experiments/${directory}/results/score-timeline.json`, import.meta.url), "utf8")), model))); const identity = (data: typeof exports[number]) => data.calls.map(call => [call.call_id, call.audio_path, call.audio_sha256, call.duration_ms]); assert.deepEqual(identity(exports[0]), identity(exports[1])); assert.deepEqual(identity(exports[0]), identity(exports[2])); assert.deepEqual(identity(exports[0]), identity(exports[3])); assert.deepEqual(identity(exports[0]), identity(exports[4])); for (const data of exports) { assert.equal(data.calls.reduce((n, call) => n + call.checkpoints.length, 0), 44); for (const call of data.calls) { assert.ok(call.points.length > 100); assert.ok(call.points.at(-1)![0] <= call.duration_ms); } } }); test("Silero checkpoints use only the last available frame, and speech is inverted once", async () => { const data = parseScoreExport(JSON.parse(await readFile(new URL( "../../../experiments/silero-vad/results/score-timeline.json", import.meta.url), "utf8")), "silero_vad"); assert.equal(data.sample_interval_ms, 32); for (const call of data.calls) { assert.equal(call.points[0][0], 32); for (const [ms] of call.points) assert.equal(ms % 32, 0); for (const [ms, score] of call.checkpoints) { const frame = scoreAt(call.points, ms, 32); assert.ok(frame); assert.equal(frame[1], score); assert.ok(frame[0] <= ms && ms - frame[0] < 32); } } }); test("LiveKit original checkpoints are real exact-time curve scores", async () => { const data = parseScoreExport(JSON.parse(await readFile(new URL( "../../../experiments/livekit-v1-mini/results/score-timeline.json", import.meta.url), "utf8")), "livekit_mini"); assert.equal(data.sample_interval_ms, 200); for (const call of data.calls) { for (const [ms, score] of call.checkpoints) { assert.deepEqual(call.points.find(point => point[0] === ms), [ms, score]); } } }); test("TurnSense curve checkpoints retain the complete probability at the exact decision", async () => { const data = parseScoreExport(JSON.parse(await readFile(new URL( "../../../experiments/turnsense-1.0/results/score-timeline.json", import.meta.url), "utf8")), "turnsense"); assert.equal(data.sample_interval_ms, 200); for (const call of data.calls) { for (const point of call.checkpoints) assert.deepEqual(call.points.find(p => p[0] === point[0]), point); } });