xiaobailing-turn-detection/labels/eot-studio/tests/model-scores.test.ts

import assert from "node:assert/strict";
import test from "node:test";
import { readFile } from "node:fs/promises";
// @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
import { parseScoreExport, scoreAt, scorePath } from "../lib/model-scores.ts";

const fixture = () => ({ schema_version: 1, model: "maai", name: "MaAI", score: "1-p_now[user]", sample_interval_ms: 100,
  calls: [{ call_id: "demo", audio_path: "demo.mp3", audio_sha256: "a".repeat(64), duration_ms: 1000,
    points: [[100, .2], [200, .9], [900, .3]], checkpoints: [[150, .8]] }] });

test("model scores accept real ordered samples and independent checkpoint times", () => {
  assert.equal(parseScoreExport(fixture(), "maai").calls[0].checkpoints[0][0], 150);
});

test("invalid exports cannot masquerade as zero scores or another model", () => {
  for (const points of [[[100, null]], [[100, 1.1]], [[100, -1]], [[100, .5], [100, .6]], [[200, .5], [100, .6]], [[1001, .5]]]) {
    const data = fixture(); data.calls[0].points = points as number[][];
    assert.throws(() => parseScoreExport(data, "maai"));
  }
  assert.throws(() => parseScoreExport(fixture(), "smart_turn"));
  const duplicate = fixture(); duplicate.calls.push(duplicate.calls[0]);
  assert.throws(() => parseScoreExport(duplicate, "maai"));
});

test("playhead values never use a future sample or carry through missing data", () => {
  const points: [number, number][] = [[100, .2], [200, .9], [900, .3]];
  assert.equal(scoreAt(points, 99, 200), null);
  assert.deepEqual(scoreAt(points, 150, 200), [100, .2]);
  assert.deepEqual(scoreAt(points, 200, 200), [200, .9]);
  assert.equal(scoreAt(points, 600, 200), null);
  assert.equal(scoreAt([], 100, 200), null);
});

test("curve gaps are not connected and steps remain on the audio clock", () => {
  const path = scorePath([[100, .2], [200, .9], [900, .3]], 1000, 500, 100, 100);
  assert.equal(path, "M50.00,72.80H100.00V19.60M450.00,65.20");
});

test("recorded inference exports agree on audio identity and retain all 44 checkpoints", async () => {
  const files = [
    ["maai", "maai-ch-kyoto"], ["smart_turn", "smart-turn-v3.2"], ["silero_vad", "silero-vad"], ["livekit_mini", "livekit-v1-mini"], ["turnsense", "turnsense-1.0"],
  ] as const;
  const exports = await Promise.all(files.map(async ([model, directory]) => parseScoreExport(
    JSON.parse(await readFile(new URL(`../../../experiments/${directory}/results/score-timeline.json`, import.meta.url), "utf8")), model)));
  const identity = (data: typeof exports[number]) => data.calls.map(call => [call.call_id, call.audio_path, call.audio_sha256, call.duration_ms]);
  assert.deepEqual(identity(exports[0]), identity(exports[1]));
  assert.deepEqual(identity(exports[0]), identity(exports[2]));
  assert.deepEqual(identity(exports[0]), identity(exports[3]));
  assert.deepEqual(identity(exports[0]), identity(exports[4]));
  for (const data of exports) {
    assert.equal(data.calls.reduce((n, call) => n + call.checkpoints.length, 0), 44);
    for (const call of data.calls) {
      assert.ok(call.points.length > 100);
      assert.ok(call.points.at(-1)![0] <= call.duration_ms);
    }
  }
});

test("Silero checkpoints use only the last available frame, and speech is inverted once", async () => {
  const data = parseScoreExport(JSON.parse(await readFile(new URL(
    "../../../experiments/silero-vad/results/score-timeline.json", import.meta.url), "utf8")), "silero_vad");
  assert.equal(data.sample_interval_ms, 32);
  for (const call of data.calls) {
    assert.equal(call.points[0][0], 32);
    for (const [ms] of call.points) assert.equal(ms % 32, 0);
    for (const [ms, score] of call.checkpoints) {
      const frame = scoreAt(call.points, ms, 32);
      assert.ok(frame);
      assert.equal(frame[1], score);
      assert.ok(frame[0] <= ms && ms - frame[0] < 32);
    }
  }
});


test("LiveKit original checkpoints are real exact-time curve scores", async () => {
  const data = parseScoreExport(JSON.parse(await readFile(new URL(
    "../../../experiments/livekit-v1-mini/results/score-timeline.json", import.meta.url), "utf8")), "livekit_mini");
  assert.equal(data.sample_interval_ms, 200);
  for (const call of data.calls) {
    for (const [ms, score] of call.checkpoints) {
      assert.deepEqual(call.points.find(point => point[0] === ms), [ms, score]);
    }
  }
});


test("TurnSense curve checkpoints retain the complete probability at the exact decision", async () => {
  const data = parseScoreExport(JSON.parse(await readFile(new URL(
    "../../../experiments/turnsense-1.0/results/score-timeline.json", import.meta.url), "utf8")), "turnsense");
  assert.equal(data.sample_interval_ms, 200);
  for (const call of data.calls) {
    for (const point of call.checkpoints) assert.deepEqual(call.points.find(p => p[0] === point[0]), point);
  }
});