新增 Mimi 与仅用户 VAP 对比并统一流式评测

labels/eot-studio/tests/model-scores.test.ts

@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
 import test from "node:test";
 import { readFile } from "node:fs/promises";
 // @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
-import { parseScoreExport, scoreAt, scorePath } from "../lib/model-scores.ts";
+import { checkpointsForDisplay, parseScoreExport, scoreAt, scorePath, type ScoreSeries } from "../lib/model-scores.ts";
 
 const fixture = () => ({ schema_version: 1, model: "maai", name: "MaAI", score: "1-p_now[user]", sample_interval_ms: 100,
   calls: [{ call_id: "demo", audio_path: "demo.mp3", audio_sha256: "a".repeat(64), duration_ms: 1000,
@@ -33,20 +33,39 @@ test("playhead values never use a future sample or carry through missing data",
 
 test("curve gaps are not connected and steps remain on the audio clock", () => {
   const path = scorePath([[100, .2], [200, .9], [900, .3]], 1000, 500, 100, 100);
-  assert.equal(path, "M50.00,72.80H100.00V19.60M450.00,65.20");
+  assert.equal(path, "M50.00,72.80H100.00V19.60H200.00M450.00,65.20H500.00");
+});
+
+test("streaming markers use current decision times, never future or stale scores", () => {
+  const data = parseScoreExport(fixture(), "maai");
+  const series = { ...data.calls[0], model: data.model, name: data.name, score: data.score,
+    sample_interval_ms: data.sample_interval_ms, checkpoint_mode: "streaming" as const };
+  assert.deepEqual(checkpointsForDisplay(series, [99, 150, 150, 200, 400, 401, 1000]),
+    [[150, .2], [200, .9], [400, .9], [1000, .3]]);
+  assert.deepEqual(checkpointsForDisplay({ ...series, checkpoint_mode: undefined }, [150]), [[150, .8]]);
+  assert.equal(scorePath([[100, .2]], 1000, 1000, 100, 100), "M100.00,72.80H300.00");
+});
+
+test("streaming exports reject independent, future or missing checkpoint sources", () => {
+  const data = { ...fixture(), checkpoint_mode: "streaming" };
+  assert.throws(() => parseScoreExport(data, "maai"), /实际评分时刻/);
+  const valid = { ...data, calls: [{ ...data.calls[0], checkpoints: [[150, .2]], checkpoint_sources: [[150, 100]] }] };
+  assert.equal(parseScoreExport(valid, "maai").calls[0].checkpoints[0][1], .2);
+  assert.throws(() => parseScoreExport({ ...valid, calls: [{ ...valid.calls[0], checkpoints: [[150, .8]] }] }, "maai"), /最后可用/);
+  assert.throws(() => parseScoreExport({ ...valid, calls: [{ ...valid.calls[0], checkpoint_sources: [[150, 200]] }] }, "maai"), /最后可用/);
 });
 
 test("recorded inference exports agree on audio identity and retain all 44 checkpoints", async () => {
   const files = [
-    ["maai", "maai-ch-kyoto"], ["smart_turn", "smart-turn-v3.2"], ["silero_vad", "silero-vad"], ["livekit_mini", "livekit-v1-mini"], ["turnsense", "turnsense-1.0"],
+    ["maai", "maai-ch-kyoto", "streaming/cpc-stereo/score-timeline.json"], ["maai_user_only", "maai-ch-kyoto", "streaming/cpc-user-only/score-timeline.json"],
+    ["maai_mimi_standard", "maai-ch-kyoto", "streaming/mimi-standard/score-timeline.json"], ["maai_mimi_mc", "maai-ch-kyoto", "streaming/mimi-mc/score-timeline.json"],
+    ["smart_turn", "smart-turn-v3.2", "score-timeline.json"], ["silero_vad", "silero-vad", "score-timeline.json"],
+    ["livekit_mini", "livekit-v1-mini", "score-timeline.json"], ["turnsense", "turnsense-1.0", "score-timeline.json"],
   ] as const;
-  const exports = await Promise.all(files.map(async ([model, directory]) => parseScoreExport(
-    JSON.parse(await readFile(new URL(`../../../experiments/${directory}/results/score-timeline.json`, import.meta.url), "utf8")), model)));
+  const exports = await Promise.all(files.map(async ([model, directory, file]) => parseScoreExport(
+    JSON.parse(await readFile(new URL(`../../../experiments/${directory}/results/${file}`, import.meta.url), "utf8")), model)));
   const identity = (data: typeof exports[number]) => data.calls.map(call => [call.call_id, call.audio_path, call.audio_sha256, call.duration_ms]);
-  assert.deepEqual(identity(exports[0]), identity(exports[1]));
-  assert.deepEqual(identity(exports[0]), identity(exports[2]));
-  assert.deepEqual(identity(exports[0]), identity(exports[3]));
-  assert.deepEqual(identity(exports[0]), identity(exports[4]));
+  for (const data of exports.slice(1)) assert.deepEqual(identity(exports[0]), identity(data));
   for (const data of exports) {
     assert.equal(data.calls.reduce((n, call) => n + call.checkpoints.length, 0), 44);
     for (const call of data.calls) {
@@ -56,6 +75,54 @@ test("recorded inference exports agree on audio identity and retain all 44 check
   }
 });
 
+test("all four VAP streaming exports, displayed circles and accuracy use the same past-only scores", async () => {
+  const sources = [["maai", "cpc-stereo"], ["maai_user_only", "cpc-user-only"],
+    ["maai_mimi_standard", "mimi-standard"], ["maai_mimi_mc", "mimi-mc"]] as const;
+  const annotationCsv = await readFile(new URL("../../EOT标注/工作数据/eot_labels.csv", import.meta.url), "utf8");
+  // @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
+  const { parseAnnotations } = await import("../lib/annotations.ts");
+  const annotations = parseAnnotations(annotationCsv);
+  for (const [model, directory] of sources) {
+    const url = new URL(`../../../experiments/maai-ch-kyoto/results/streaming/${directory}/`, import.meta.url);
+    const data = parseScoreExport(JSON.parse(await readFile(new URL("score-timeline.json", url), "utf8")), model);
+    const summary = JSON.parse(await readFile(new URL("summary.json", url), "utf8"));
+    assert.equal(data.checkpoint_mode, "streaming");
+    let correct = 0, count = 0;
+    for (const call of data.calls) {
+      const labels = annotations.filter(row => row.call_id === call.call_id);
+      const series: ScoreSeries = { ...call, model, name: data.name, score: data.score,
+        sample_interval_ms: data.sample_interval_ms, checkpoint_mode: data.checkpoint_mode };
+      assert.deepEqual(checkpointsForDisplay(series, labels.map(row => row.decision_ms)), call.checkpoints);
+      for (const [ms, score] of call.checkpoints) {
+        assert.equal(scoreAt(call.points, ms, data.sample_interval_ms * 2)![1], score);
+        const label = labels.find(row => row.decision_ms === ms)!;
+        if (label.can_reply_now !== "") { count += 1; correct += Number(Number(score > .5) === Number(label.can_reply_now)); }
+      }
+    }
+    assert.equal(summary.main_metrics.n, count);
+    assert.equal(summary.main_metrics.correct, correct);
+    assert.equal(summary.main_metrics.accuracy, correct / count);
+  }
+});
+
+test("user-only VAP checkpoints use the current labels and the silent-robot experiment", async () => {
+  const load = async (file: string) => JSON.parse(await readFile(new URL(
+    `../../../experiments/maai-ch-kyoto/results/user-only/${file}`, import.meta.url), "utf8"));
+  const data = parseScoreExport(await load("score-timeline.json"), "maai_user_only");
+  const summary = await load("summary.json");
+  const annotationCsv = await readFile(new URL("../../EOT标注/工作数据/eot_labels.csv", import.meta.url), "utf8");
+  // @ts-expect-error Node's native TypeScript runner resolves explicit .ts imports.
+  const { parseAnnotations } = await import("../lib/annotations.ts");
+  const annotations = parseAnnotations(annotationCsv).filter(row => row.can_reply_now !== "");
+  assert.equal(summary.witness.robot_and_future_invariance, true);
+  assert.equal(summary.witness.repeat_identical, true);
+  assert.equal(data.sample_interval_ms, 100);
+  const checkpoints = data.calls.flatMap(call => call.checkpoints.map(([ms]) => `${call.call_id}:${ms}`)).sort();
+  assert.deepEqual(checkpoints, annotations.map(row => `${row.call_id}:${row.decision_ms}`).sort());
+  assert.equal(summary.main_metrics.n, annotations.length);
+  assert.equal(summary.same_point_stereo_reference.n, annotations.length);
+});
+
 test("Silero checkpoints use only the last available frame, and speech is inverted once", async () => {
   const data = parseScoreExport(JSON.parse(await readFile(new URL(
     "../../../experiments/silero-vad/results/score-timeline.json", import.meta.url), "utf8")), "silero_vad");
@@ -72,6 +139,31 @@ test("Silero checkpoints use only the last available frame, and speech is invert
   }
 });
 
+test("Mimi standard and MC use verified distinct checkpoints with the same causal stereo inputs", async () => {
+  const load = async (directory: string, file: string) => JSON.parse(await readFile(new URL(
+    `../../../experiments/maai-ch-kyoto/results/${directory}/${file}`, import.meta.url), "utf8"));
+  const standard = await load("mimi-standard", "summary.json");
+  const mc = await load("mimi-mc", "summary.json");
+  const checkpointKeys = [];
+  for (const [id, dir, summary] of [["maai_mimi_standard", "mimi-standard", standard], ["maai_mimi_mc", "mimi-mc", mc]] as const) {
+    const data = parseScoreExport(await load(dir, "score-timeline.json"), id);
+    assert.equal(data.sample_interval_ms, 80);
+    assert.equal(summary.witness.repeat_identical, true);
+    assert.equal(summary.witness.future_invariance, true);
+    assert.equal(summary.witness.both_channels_nonzero, true);
+    assert.deepEqual(summary.witness.loading_audit.missing_trainable_parameters, []);
+    assert.deepEqual(summary.witness.loading_audit.mismatched_parameters, []);
+    assert.equal(summary.provenance.channels["1"], "recorded robot");
+    checkpointKeys.push(data.calls.flatMap(call => call.checkpoints.map(([ms]) => `${call.call_id}:${ms}`)).sort());
+  }
+  assert.equal(standard.provenance.mc, false);
+  assert.equal(mc.provenance.mc, true);
+  assert.notEqual(standard.provenance.checkpoint_filename, mc.provenance.checkpoint_filename);
+  assert.equal(standard.provenance.label_sha256, mc.provenance.label_sha256);
+  assert.deepEqual(standard.provenance.audio_sha256, mc.provenance.audio_sha256);
+  assert.deepEqual(checkpointKeys[0], checkpointKeys[1]);
+});
+
 
 test("LiveKit original checkpoints are real exact-time curve scores", async () => {
   const data = parseScoreExport(JSON.parse(await readFile(new URL(

labels/eot-studio/tests/score-preferences.test.ts

@@ -7,6 +7,8 @@ test("first visit starts with three comparison curves and each model's reference
   const defaults = readScorePreferences(null);
   assert.deepEqual(Object.entries(defaults.visibility).filter(([, value]) => value).map(([id]) => id), ["maai", "smart_turn", "turnsense"]);
   assert.equal(defaults.thresholds.livekit_mini, .355);
+  assert.equal(defaults.visibility.maai_user_only, false);
+  assert.equal(defaults.thresholds.maai_user_only, .5);
 });
 
 test("saved choices preserve clear-all and accept only supported reference thresholds", () => {
@@ -26,3 +28,22 @@ test("partial and invalid preferences leave future models on their defaults", ()
   assert.equal("unknown" in saved.visibility, false);
   assert.deepEqual(readScorePreferences({ version: 999, visibility: { maai: false } }), readScorePreferences(null));
 });
+
+test("user-only VAP has independent visibility and thresholds alongside stereo VAP", () => {
+  const saved = readScorePreferences({ version: 1, visibility: { maai: false, maai_user_only: true },
+    thresholds: { maai: .8, maai_user_only: .11 } });
+  assert.equal(saved.visibility.maai, false);
+  assert.equal(saved.visibility.maai_user_only, true);
+  assert.equal(saved.thresholds.maai, .8);
+  assert.equal(saved.thresholds.maai_user_only, .11);
+});
+
+test("Mimi standard and noise-robust comparison settings are independent", () => {
+  const saved = readScorePreferences({ version: 1,
+    visibility: { maai_mimi_standard: true, maai_mimi_mc: false },
+    thresholds: { maai_mimi_standard: .8, maai_mimi_mc: .11 } });
+  assert.equal(saved.visibility.maai_mimi_standard, true);
+  assert.equal(saved.visibility.maai_mimi_mc, false);
+  assert.equal(saved.thresholds.maai_mimi_standard, .8);
+  assert.equal(saved.thresholds.maai_mimi_mc, .11);
+});