188 lines
11 KiB
TypeScript
188 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { readFile } from "node:fs/promises";
|
|
// @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
|
|
import { checkpointsForDisplay, parseScoreExport, scoreAt, scorePath, type ScoreSeries } from "../lib/model-scores.ts";
|
|
|
|
const fixture = () => ({ schema_version: 1, model: "maai", name: "MaAI", score: "1-p_now[user]", sample_interval_ms: 100,
|
|
calls: [{ call_id: "demo", audio_path: "demo.mp3", audio_sha256: "a".repeat(64), duration_ms: 1000,
|
|
points: [[100, .2], [200, .9], [900, .3]], checkpoints: [[150, .8]] }] });
|
|
|
|
test("model scores accept real ordered samples and independent checkpoint times", () => {
|
|
assert.equal(parseScoreExport(fixture(), "maai").calls[0].checkpoints[0][0], 150);
|
|
});
|
|
|
|
test("invalid exports cannot masquerade as zero scores or another model", () => {
|
|
for (const points of [[[100, null]], [[100, 1.1]], [[100, -1]], [[100, .5], [100, .6]], [[200, .5], [100, .6]], [[1001, .5]]]) {
|
|
const data = fixture(); data.calls[0].points = points as number[][];
|
|
assert.throws(() => parseScoreExport(data, "maai"));
|
|
}
|
|
assert.throws(() => parseScoreExport(fixture(), "smart_turn"));
|
|
const duplicate = fixture(); duplicate.calls.push(duplicate.calls[0]);
|
|
assert.throws(() => parseScoreExport(duplicate, "maai"));
|
|
});
|
|
|
|
test("playhead values never use a future sample or carry through missing data", () => {
|
|
const points: [number, number][] = [[100, .2], [200, .9], [900, .3]];
|
|
assert.equal(scoreAt(points, 99, 200), null);
|
|
assert.deepEqual(scoreAt(points, 150, 200), [100, .2]);
|
|
assert.deepEqual(scoreAt(points, 200, 200), [200, .9]);
|
|
assert.equal(scoreAt(points, 600, 200), null);
|
|
assert.equal(scoreAt([], 100, 200), null);
|
|
});
|
|
|
|
test("curve gaps are not connected and steps remain on the audio clock", () => {
|
|
const path = scorePath([[100, .2], [200, .9], [900, .3]], 1000, 500, 100, 100);
|
|
assert.equal(path, "M50.00,72.80H100.00V19.60H200.00M450.00,65.20H500.00");
|
|
});
|
|
|
|
test("streaming markers use current decision times, never future or stale scores", () => {
|
|
const data = parseScoreExport(fixture(), "maai");
|
|
const series = { ...data.calls[0], model: data.model, name: data.name, score: data.score,
|
|
sample_interval_ms: data.sample_interval_ms, checkpoint_mode: "streaming" as const };
|
|
assert.deepEqual(checkpointsForDisplay(series, [99, 150, 150, 200, 400, 401, 1000]),
|
|
[[150, .2], [200, .9], [400, .9], [1000, .3]]);
|
|
assert.deepEqual(checkpointsForDisplay({ ...series, checkpoint_mode: undefined }, [150]), [[150, .8]]);
|
|
assert.equal(scorePath([[100, .2]], 1000, 1000, 100, 100), "M100.00,72.80H300.00");
|
|
});
|
|
|
|
test("streaming exports reject independent, future or missing checkpoint sources", () => {
|
|
const data = { ...fixture(), checkpoint_mode: "streaming" };
|
|
assert.throws(() => parseScoreExport(data, "maai"), /实际评分时刻/);
|
|
const valid = { ...data, calls: [{ ...data.calls[0], checkpoints: [[150, .2]], checkpoint_sources: [[150, 100]] }] };
|
|
assert.equal(parseScoreExport(valid, "maai").calls[0].checkpoints[0][1], .2);
|
|
assert.throws(() => parseScoreExport({ ...valid, calls: [{ ...valid.calls[0], checkpoints: [[150, .8]] }] }, "maai"), /最后可用/);
|
|
assert.throws(() => parseScoreExport({ ...valid, calls: [{ ...valid.calls[0], checkpoint_sources: [[150, 200]] }] }, "maai"), /最后可用/);
|
|
});
|
|
|
|
test("recorded inference exports agree on audio identity and retain all 44 checkpoints", async () => {
|
|
const files = [
|
|
["maai", "maai-ch-kyoto", "streaming/cpc-stereo/score-timeline.json"], ["maai_user_only", "maai-ch-kyoto", "streaming/cpc-user-only/score-timeline.json"],
|
|
["maai_mimi_standard", "maai-ch-kyoto", "streaming/mimi-standard/score-timeline.json"], ["maai_mimi_mc", "maai-ch-kyoto", "streaming/mimi-mc/score-timeline.json"],
|
|
["smart_turn", "smart-turn-v3.2", "score-timeline.json"], ["silero_vad", "silero-vad", "score-timeline.json"],
|
|
["livekit_mini", "livekit-v1-mini", "score-timeline.json"], ["turnsense", "turnsense-1.0", "score-timeline.json"],
|
|
] as const;
|
|
const exports = await Promise.all(files.map(async ([model, directory, file]) => parseScoreExport(
|
|
JSON.parse(await readFile(new URL(`../../../experiments/${directory}/results/${file}`, import.meta.url), "utf8")), model)));
|
|
const identity = (data: typeof exports[number]) => data.calls.map(call => [call.call_id, call.audio_path, call.audio_sha256, call.duration_ms]);
|
|
for (const data of exports.slice(1)) assert.deepEqual(identity(exports[0]), identity(data));
|
|
for (const data of exports) {
|
|
assert.equal(data.calls.reduce((n, call) => n + call.checkpoints.length, 0), 44);
|
|
for (const call of data.calls) {
|
|
assert.ok(call.points.length > 100);
|
|
assert.ok(call.points.at(-1)![0] <= call.duration_ms);
|
|
}
|
|
}
|
|
});
|
|
|
|
test("all four VAP streaming exports, displayed circles and accuracy use the same past-only scores", async () => {
|
|
const sources = [["maai", "cpc-stereo"], ["maai_user_only", "cpc-user-only"],
|
|
["maai_mimi_standard", "mimi-standard"], ["maai_mimi_mc", "mimi-mc"]] as const;
|
|
const annotationCsv = await readFile(new URL("../../EOT标注/工作数据/eot_labels.csv", import.meta.url), "utf8");
|
|
// @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
|
|
const { parseAnnotations } = await import("../lib/annotations.ts");
|
|
const annotations = parseAnnotations(annotationCsv);
|
|
for (const [model, directory] of sources) {
|
|
const url = new URL(`../../../experiments/maai-ch-kyoto/results/streaming/${directory}/`, import.meta.url);
|
|
const data = parseScoreExport(JSON.parse(await readFile(new URL("score-timeline.json", url), "utf8")), model);
|
|
const summary = JSON.parse(await readFile(new URL("summary.json", url), "utf8"));
|
|
assert.equal(data.checkpoint_mode, "streaming");
|
|
let correct = 0, count = 0;
|
|
for (const call of data.calls) {
|
|
const labels = annotations.filter(row => row.call_id === call.call_id);
|
|
const series: ScoreSeries = { ...call, model, name: data.name, score: data.score,
|
|
sample_interval_ms: data.sample_interval_ms, checkpoint_mode: data.checkpoint_mode };
|
|
assert.deepEqual(checkpointsForDisplay(series, labels.map(row => row.decision_ms)), call.checkpoints);
|
|
for (const [ms, score] of call.checkpoints) {
|
|
assert.equal(scoreAt(call.points, ms, data.sample_interval_ms * 2)![1], score);
|
|
const label = labels.find(row => row.decision_ms === ms)!;
|
|
if (label.can_reply_now !== "") { count += 1; correct += Number(Number(score > .5) === Number(label.can_reply_now)); }
|
|
}
|
|
}
|
|
assert.equal(summary.main_metrics.n, count);
|
|
assert.equal(summary.main_metrics.correct, correct);
|
|
assert.equal(summary.main_metrics.accuracy, correct / count);
|
|
}
|
|
});
|
|
|
|
test("user-only VAP checkpoints use the current labels and the silent-robot experiment", async () => {
|
|
const load = async (file: string) => JSON.parse(await readFile(new URL(
|
|
`../../../experiments/maai-ch-kyoto/results/user-only/${file}`, import.meta.url), "utf8"));
|
|
const data = parseScoreExport(await load("score-timeline.json"), "maai_user_only");
|
|
const summary = await load("summary.json");
|
|
const annotationCsv = await readFile(new URL("../../EOT标注/工作数据/eot_labels.csv", import.meta.url), "utf8");
|
|
// @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
|
|
const { parseAnnotations } = await import("../lib/annotations.ts");
|
|
const annotations = parseAnnotations(annotationCsv).filter(row => row.can_reply_now !== "");
|
|
assert.equal(summary.witness.robot_and_future_invariance, true);
|
|
assert.equal(summary.witness.repeat_identical, true);
|
|
assert.equal(data.sample_interval_ms, 100);
|
|
const checkpoints = data.calls.flatMap(call => call.checkpoints.map(([ms]) => `${call.call_id}:${ms}`)).sort();
|
|
assert.deepEqual(checkpoints, annotations.map(row => `${row.call_id}:${row.decision_ms}`).sort());
|
|
assert.equal(summary.main_metrics.n, annotations.length);
|
|
assert.equal(summary.same_point_stereo_reference.n, annotations.length);
|
|
});
|
|
|
|
test("Silero checkpoints use only the last available frame, and speech is inverted once", async () => {
|
|
const data = parseScoreExport(JSON.parse(await readFile(new URL(
|
|
"../../../experiments/silero-vad/results/score-timeline.json", import.meta.url), "utf8")), "silero_vad");
|
|
assert.equal(data.sample_interval_ms, 32);
|
|
for (const call of data.calls) {
|
|
assert.equal(call.points[0][0], 32);
|
|
for (const [ms] of call.points) assert.equal(ms % 32, 0);
|
|
for (const [ms, score] of call.checkpoints) {
|
|
const frame = scoreAt(call.points, ms, 32);
|
|
assert.ok(frame);
|
|
assert.equal(frame[1], score);
|
|
assert.ok(frame[0] <= ms && ms - frame[0] < 32);
|
|
}
|
|
}
|
|
});
|
|
|
|
test("Mimi standard and MC use verified distinct checkpoints with the same causal stereo inputs", async () => {
|
|
const load = async (directory: string, file: string) => JSON.parse(await readFile(new URL(
|
|
`../../../experiments/maai-ch-kyoto/results/${directory}/${file}`, import.meta.url), "utf8"));
|
|
const standard = await load("mimi-standard", "summary.json");
|
|
const mc = await load("mimi-mc", "summary.json");
|
|
const checkpointKeys = [];
|
|
for (const [id, dir, summary] of [["maai_mimi_standard", "mimi-standard", standard], ["maai_mimi_mc", "mimi-mc", mc]] as const) {
|
|
const data = parseScoreExport(await load(dir, "score-timeline.json"), id);
|
|
assert.equal(data.sample_interval_ms, 80);
|
|
assert.equal(summary.witness.repeat_identical, true);
|
|
assert.equal(summary.witness.future_invariance, true);
|
|
assert.equal(summary.witness.both_channels_nonzero, true);
|
|
assert.deepEqual(summary.witness.loading_audit.missing_trainable_parameters, []);
|
|
assert.deepEqual(summary.witness.loading_audit.mismatched_parameters, []);
|
|
assert.equal(summary.provenance.channels["1"], "recorded robot");
|
|
checkpointKeys.push(data.calls.flatMap(call => call.checkpoints.map(([ms]) => `${call.call_id}:${ms}`)).sort());
|
|
}
|
|
assert.equal(standard.provenance.mc, false);
|
|
assert.equal(mc.provenance.mc, true);
|
|
assert.notEqual(standard.provenance.checkpoint_filename, mc.provenance.checkpoint_filename);
|
|
assert.equal(standard.provenance.label_sha256, mc.provenance.label_sha256);
|
|
assert.deepEqual(standard.provenance.audio_sha256, mc.provenance.audio_sha256);
|
|
assert.deepEqual(checkpointKeys[0], checkpointKeys[1]);
|
|
});
|
|
|
|
|
|
test("LiveKit original checkpoints are real exact-time curve scores", async () => {
|
|
const data = parseScoreExport(JSON.parse(await readFile(new URL(
|
|
"../../../experiments/livekit-v1-mini/results/score-timeline.json", import.meta.url), "utf8")), "livekit_mini");
|
|
assert.equal(data.sample_interval_ms, 200);
|
|
for (const call of data.calls) {
|
|
for (const [ms, score] of call.checkpoints) {
|
|
assert.deepEqual(call.points.find(point => point[0] === ms), [ms, score]);
|
|
}
|
|
}
|
|
});
|
|
|
|
|
|
test("TurnSense curve checkpoints retain the complete probability at the exact decision", async () => {
|
|
const data = parseScoreExport(JSON.parse(await readFile(new URL(
|
|
"../../../experiments/turnsense-1.0/results/score-timeline.json", import.meta.url), "utf8")), "turnsense");
|
|
assert.equal(data.sample_interval_ms, 200);
|
|
for (const call of data.calls) {
|
|
for (const point of call.checkpoints) assert.deepEqual(call.points.find(p => p[0] === point[0]), point);
|
|
}
|
|
});
|