记录三模型话轮检测实验、44点评估及延迟对比
This commit is contained in:
Xin Wang
@@ -0,0 +1,3 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.onnx
|
||||
@@ -0,0 +1 @@
|
||||
3.11
|
||||
@@ -0,0 +1,24 @@
|
||||
BSD 2-Clause License
|
||||
|
||||
Copyright (c) 2024–2025, Daily
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -0,0 +1,13 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000):
|
||||
"""Truncate audio to last n seconds or pad with zeros to meet n seconds."""
|
||||
max_samples = n_seconds * sample_rate
|
||||
if len(audio_array) > max_samples:
|
||||
return audio_array[-max_samples:]
|
||||
elif len(audio_array) < max_samples:
|
||||
# Pad with zeros at the beginning
|
||||
padding = max_samples - len(audio_array)
|
||||
return np.pad(audio_array, (padding, 0), mode='constant', constant_values=0)
|
||||
return audio_array
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Benchmark the actual Pipecat Smart Turn endpoint call on the 44 inputs."""
|
||||
import csv
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
os.environ.setdefault("HF_HOME", str(HERE / ".cache/huggingface"))
|
||||
os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
|
||||
os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from loguru import logger
|
||||
import pipecat.audio.turn.smart_turn.local_smart_turn_v3 as local
|
||||
|
||||
|
||||
def stats(values):
|
||||
v = np.asarray(values, dtype=float)
|
||||
return dict(n=len(v), mean_ms=float(v.mean()), median_ms=float(np.median(v)),
|
||||
p95_ms=float(np.percentile(v, 95)), p99_ms=float(np.percentile(v, 99)),
|
||||
min_ms=float(v.min()), max_ms=float(v.max()))
|
||||
|
||||
|
||||
class TimedSession:
|
||||
def __init__(self, session, current):
|
||||
self.session, self.current = session, current
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
begin = time.perf_counter()
|
||||
output = self.session.run(*args, **kwargs)
|
||||
self.current["onnx_ms"] = (time.perf_counter()-begin)*1000
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
logger.remove()
|
||||
with (HERE / "results/work-data-can-reply/predictions.csv").open(encoding="utf-8-sig") as f:
|
||||
labels = list(csv.DictReader(f))
|
||||
audio_cache, inputs = {}, []
|
||||
for row in labels:
|
||||
call = row["call_id"]
|
||||
if call not in audio_cache:
|
||||
pcm, sr = sf.read(ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3", dtype="int16", always_2d=True)
|
||||
audio_cache[call] = pcm[:, 0].astype(np.float32)/32768.0, sr
|
||||
pcm, sr = audio_cache[call]
|
||||
start = round(float(row["audio_start_ms"])*sr/1000)
|
||||
end = int(row["decision_ms"])*sr//1000
|
||||
inputs.append((row, np.ascontiguousarray(pcm[start:end]), sr))
|
||||
report = dict(method="Warm cached model endpoint timing; same 44 Smart Turn evaluation crops, 5 repetitions. Includes resampling, left padding, Whisper features and ONNX inference. Excludes audio decoding, import, model loading and silence/ASR waits.",
|
||||
pipecat_version=importlib.metadata.version("pipecat-ai"),
|
||||
onnxruntime_version=importlib.metadata.version("onnxruntime"),
|
||||
repetitions=5, cases={})
|
||||
original_feature = local.compute_whisper_log_mel_features
|
||||
for threads in (1, 4):
|
||||
began = time.perf_counter()
|
||||
analyzer = local.LocalSmartTurnAnalyzerV3(smart_turn_model_path=str(HERE / "smart-turn-v3.2-cpu.onnx"), cpu_count=threads)
|
||||
load_ms = (time.perf_counter()-began)*1000
|
||||
analyzer.set_sample_rate(inputs[0][2])
|
||||
for _ in range(5):
|
||||
analyzer._predict_endpoint(inputs[0][1])
|
||||
current = {}
|
||||
original_resample = analyzer._resample_to_model_rate
|
||||
|
||||
def resample(audio):
|
||||
began = time.perf_counter()
|
||||
output = original_resample(audio)
|
||||
current["resample_ms"] = (time.perf_counter()-began)*1000
|
||||
return output
|
||||
|
||||
def features(*args, **kwargs):
|
||||
began = time.perf_counter()
|
||||
output = original_feature(*args, **kwargs)
|
||||
current["feature_ms"] = (time.perf_counter()-began)*1000
|
||||
return output
|
||||
|
||||
analyzer._resample_to_model_rate = resample
|
||||
analyzer._session = TimedSession(analyzer._session, current)
|
||||
local.compute_whisper_log_mel_features = features
|
||||
measured = []
|
||||
try:
|
||||
for repeat in range(5):
|
||||
# Rotate order to avoid always timing the same input first.
|
||||
order = inputs[repeat:] + inputs[:repeat]
|
||||
for label, sample, sr in order:
|
||||
analyzer.set_sample_rate(sr)
|
||||
current.clear()
|
||||
began = time.perf_counter()
|
||||
result = analyzer._predict_endpoint(sample)
|
||||
total = (time.perf_counter()-began)*1000
|
||||
assert abs(result["probability"]-float(label["probability"])) < 1e-4, (label["call_id"], label["decision_ms"], result["probability"], label["probability"])
|
||||
measured.append(dict(call_id=label["call_id"], decision_ms=int(label["decision_ms"]),
|
||||
repetition=repeat, total_ms=total, **current))
|
||||
finally:
|
||||
local.compute_whisper_log_mel_features = original_feature
|
||||
report["cases"][str(threads)] = dict(cpu_threads=threads, model_load_ms=load_ms,
|
||||
total=stats([r["total_ms"] for r in measured]),
|
||||
resample=stats([r["resample_ms"] for r in measured]),
|
||||
features=stats([r["feature_ms"] for r in measured]),
|
||||
onnx=stats([r["onnx_ms"] for r in measured]), samples=measured)
|
||||
print(threads, json.dumps({k:v for k,v in report["cases"][str(threads)].items() if k != "samples"}), flush=True)
|
||||
destination = HERE / "results/latency.json"
|
||||
destination.write_text(json.dumps(report, indent=2)+"\n")
|
||||
print(f"Saved {destination}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Classify each annotation independently without end-signal state.
|
||||
|
||||
Past ASR segments locate the current user speech block; continuations separated
|
||||
by <=3 s keep the block start. No label value or earlier model prediction is
|
||||
used to construct inputs. For missing ASR use past Silero VAD starts only.
|
||||
"""
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from loguru import logger
|
||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
OUT = HERE / "results/work-data-can-reply"
|
||||
|
||||
|
||||
def read(path):
|
||||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
|
||||
def write(path, rows):
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def metrics(rows):
|
||||
c = Counter((int(r["can_reply_now"]), r["prediction"]) for r in rows
|
||||
if r["can_reply_now"] in ("0", "1") and r["prediction"] is not None)
|
||||
tn, fp, fn, tp = c[0, 0], c[0, 1], c[1, 0], c[1, 1]
|
||||
total = tn + fp + fn + tp
|
||||
ratio = lambda n, d: n/d if d else None
|
||||
return dict(n=total, tn=tn, fp=fp, fn=fn, tp=tp,
|
||||
accuracy=ratio(tn+tp, total), false_reply_rate=ratio(fp, fp+tn),
|
||||
false_hold_rate=ratio(fn, fn+tp), recall=ratio(tp, tp+fn),
|
||||
precision=ratio(tp, tp+fp), f1=ratio(2*tp, 2*tp+fp+fn))
|
||||
|
||||
|
||||
def block_origin(history, decision):
|
||||
# Use only segments that have begun by this decision. Full ASR end times
|
||||
# are only used for PREVIOUS segments that have already ended.
|
||||
users = sorted((r for r in history if r["role"] == "user"
|
||||
and int(r["start_ms"]) < decision), key=lambda r: int(r["start_ms"]))
|
||||
if not users:
|
||||
return None
|
||||
index = len(users)-1
|
||||
while index > 0:
|
||||
previous_end = int(users[index-1]["end_ms"])
|
||||
start = int(users[index]["start_ms"])
|
||||
if previous_end > decision or start-previous_end > 3000:
|
||||
break
|
||||
# A complete non-overlapping robot question between user segments
|
||||
# establishes a new observed conversational block.
|
||||
if any(r["role"] == "agent" and previous_end <= int(r["start_ms"])
|
||||
and int(r["end_ms"]) <= start for r in history):
|
||||
break
|
||||
index -= 1
|
||||
return max(0, int(users[index]["start_ms"])-500)
|
||||
|
||||
|
||||
def main():
|
||||
logger.remove()
|
||||
labels_path = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
||||
history_path = ROOT / "labels/EOT标注/工作数据/asr_history.csv"
|
||||
labels, history = read(labels_path), read(history_path)
|
||||
events = read(HERE / "results/work-data-pipecat/events.csv")
|
||||
analyzer = LocalSmartTurnAnalyzerV3(
|
||||
smart_turn_model_path=str(HERE / "smart-turn-v3.2-cpu.onnx"), cpu_count=1)
|
||||
audio = {}
|
||||
rows = []
|
||||
for label in labels:
|
||||
call = label["call_id"]
|
||||
if call not in audio:
|
||||
pcm, sr = sf.read(ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3",
|
||||
dtype="int16", always_2d=True)
|
||||
audio[call] = (pcm[:, 0], sr)
|
||||
pcm, sr = audio[call]
|
||||
decision = int(label["decision_ms"])
|
||||
origin = block_origin([r for r in history if r["call_id"] == call], decision)
|
||||
source = "asr_user_block"
|
||||
issue = ""
|
||||
if origin is None:
|
||||
# VAD events are independent of Smart Turn acceptance/rejection.
|
||||
# Use only a start observed BEFORE the annotation, no EOT events.
|
||||
starts = [float(e["event_ms"]) for e in events if e["call_id"] == call
|
||||
and e["event"] == "vad_start" and float(e["event_ms"]) <= decision]
|
||||
if not starts:
|
||||
raise ValueError(f"No causal user/VAD onset for {call} {decision}")
|
||||
origin = max(0, starts[-1] + 32 - 200 - 500)
|
||||
source = "past_vad_missing_asr"
|
||||
issue = "missing_earlier_user_asr_segment"
|
||||
start = max(round(origin*sr/1000), decision*sr//1000 - sr*8)
|
||||
end = decision*sr//1000
|
||||
assert 0 <= start < end <= len(pcm)
|
||||
samples = pcm[start:end].astype(np.float32)/32768.0
|
||||
analyzer.set_sample_rate(sr)
|
||||
result = analyzer._predict_endpoint(samples)
|
||||
r = dict(label)
|
||||
r.update(probability=result["probability"], prediction=result["prediction"],
|
||||
correct=int(result["prediction"] == int(label["can_reply_now"])),
|
||||
error_type=("false_reply" if result["prediction"] else "false_hold")
|
||||
if result["prediction"] != int(label["can_reply_now"]) else "correct",
|
||||
audio_start_ms=start*1000/sr, audio_end_ms=decision,
|
||||
real_audio_ms=(end-start)*1000/sr, model_input_ms=8000,
|
||||
leading_zero_ms=8000-(end-start)*1000/sr,
|
||||
user_block_origin_ms=origin, input_source=source, alignment_issue=issue,
|
||||
audio_channel=0, label_status="draft" if label["annotator"] == "xlsx_draft"
|
||||
else "unconfirmed_manual")
|
||||
rows.append(r)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
write(OUT / "predictions.csv", rows)
|
||||
write(OUT / "errors.csv", [r for r in rows if not r["correct"]])
|
||||
thresholds = []
|
||||
for threshold in (0.5, 0.7, 0.8, 0.9, 0.95, 0.98, 0.99):
|
||||
recomputed = [dict(r, prediction=int(r["probability"] > threshold)) for r in rows]
|
||||
thresholds.append(dict(threshold=threshold, **metrics(recomputed)))
|
||||
write(OUT / "thresholds.csv", thresholds)
|
||||
summary = dict(total_points=len(rows), metrics=metrics(rows),
|
||||
grouped={field: {value: metrics([r for r in rows if r[field] == value])
|
||||
for value in sorted({r[field] for r in rows})}
|
||||
for field in ("label_status", "call_id", "reason", "input_source")},
|
||||
config=dict(pipecat_version=importlib.metadata.version("pipecat-ai"),
|
||||
model="smart-turn-v3.2-cpu.onnx", threshold=0.5,
|
||||
pre_speech_ms=500, max_duration_secs=8, continuation_gap_ms=3000,
|
||||
selected_audio_channel=0,
|
||||
model_sha256=hashlib.sha256((HERE/"smart-turn-v3.2-cpu.onnx").read_bytes()).hexdigest(),
|
||||
label_sha256=hashlib.sha256(labels_path.read_bytes()).hexdigest(),
|
||||
history_sha256=hashlib.sha256(history_path.read_bytes()).hexdigest()),
|
||||
method="Each exact decision_ms is classified directly by the model. Earlier model scores, EOT signals, VAD stop gates, silence timeout forced replies and ASR return gates never determine prediction or clear this input.",
|
||||
threshold_diagnostics=thresholds,
|
||||
audio_origin="Past ASR user segments locate current speech block. Consecutive user segments within 3 s share the first start unless a complete non-overlapping robot question separates them. Keep 500 ms before that start and then last 8 s. Intra-segment pauses are not resolved by coarse ASR timing. Two missing-ASR points use only past Silero VAD starts; full-audio model events are not used.",
|
||||
limitations=["Exploratory scoring: 29 xlsx_draft and 15 rows missing annotator/confidence; no explicit verified status.",
|
||||
"Coarse ASR speech block origin approximates the discussed cache rule; this is point classification, not a complete online Pipecat strategy replay.",
|
||||
"cb3238dc 13540 text is 辽宁葫芦岛 but its corresponding user ASR starts at 16600; review alignment. e304f740 11531 surname utterance is absent from ASR history.",
|
||||
"User channel 0 can contain robot echo/overlap; no future ASR-based masking.",
|
||||
"Selected-point false-reply rate is not interval-level interruption rate."])
|
||||
(OUT/"summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2)+"\n")
|
||||
print(json.dumps(summary["metrics"], ensure_ascii=False, indent=2))
|
||||
for r in rows:
|
||||
if not r["correct"]:
|
||||
print(r["call_id"][:8], r["decision_ms"], r["can_reply_now"],
|
||||
round(r["probability"], 5), r["audio_start_ms"], r["user_text_at_decision"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Causal, audio-only Pipecat replay and non-mutating annotation-point probes.
|
||||
|
||||
The real Pipecat VAD and turn analyzer run on a media clock. Frames are 32 ms.
|
||||
At annotated times an additional diagnostic model query does not alter replay.
|
||||
No labels, transcripts or future audio are given to either neural model.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
import json
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from loguru import logger
|
||||
import pipecat.audio.turn.smart_turn.base_smart_turn as base_module
|
||||
import pipecat.audio.vad.silero as silero_module
|
||||
from pipecat.audio.turn.base_turn_analyzer import EndOfTurnState
|
||||
from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
|
||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.audio.vad.vad_analyzer import VADParams, VADState
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
|
||||
|
||||
def digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def read_csv(path):
|
||||
with path.open(encoding="utf-8-sig", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def write_csv(path, rows):
|
||||
if not rows:
|
||||
return
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def metrics(rows, prediction):
|
||||
selected = [r for r in rows if r["input_valid"] and r[prediction] in (0, 1)
|
||||
and r["can_reply_now"] in ("0", "1")]
|
||||
counts = Counter((int(r["can_reply_now"]), r[prediction]) for r in selected)
|
||||
tn, fp, fn, tp = (counts[(0, 0)], counts[(0, 1)], counts[(1, 0)], counts[(1, 1)])
|
||||
ratio = lambda n, d: n / d if d else None
|
||||
return dict(n=len(selected), tn=tn, fp=fp, fn=fn, tp=tp,
|
||||
accuracy=ratio(tp+tn, len(selected)), false_reply_rate=ratio(fp, tn+fp),
|
||||
false_hold_rate=ratio(fn, tp+fn), recall=ratio(tp, tp+fn),
|
||||
precision=ratio(tp, tp+fp), f1=ratio(2*tp, 2*tp+fp+fn))
|
||||
|
||||
|
||||
class MediaClock:
|
||||
seconds = 0.0
|
||||
|
||||
def time(self):
|
||||
return 1000.0 + self.seconds
|
||||
|
||||
def monotonic(self):
|
||||
# Non-zero origin matches BaseSmartTurn's 0 sentinel.
|
||||
return 1000.0 + self.seconds
|
||||
|
||||
|
||||
def source_audio_start(analyzer):
|
||||
"""Return the exact selected buffer-frame start before max-duration trim."""
|
||||
target = analyzer._speech_start_time - (
|
||||
analyzer.params.pre_speech_ms / 1000 + analyzer._vad_start_secs)
|
||||
times = [t for t, _ in analyzer._audio_buffer]
|
||||
if not times:
|
||||
return None
|
||||
# BaseSmartTurn's chunks are timestamped at arrival/end, not frame start.
|
||||
index = next((i for i, t in enumerate(times) if t >= target), 0)
|
||||
t, chunk = analyzer._audio_buffer[index]
|
||||
return max(0.0, t - 1000 - len(chunk) / analyzer.sample_rate)
|
||||
|
||||
|
||||
def replay(call_id, labels, history, audio_path, clock, events, channel):
|
||||
audio, sr = sf.read(audio_path, dtype="int16", always_2d=True)
|
||||
if sr not in (8000, 16000) or channel >= audio.shape[1]:
|
||||
raise ValueError(f"Unsupported audio format: {audio_path}")
|
||||
pcm = audio[:, channel]
|
||||
vad = SileroVADAnalyzer(sample_rate=sr, params=VADParams())
|
||||
vad.set_sample_rate(sr)
|
||||
analyzer = LocalSmartTurnAnalyzerV3(
|
||||
smart_turn_model_path=str(HERE / "smart-turn-v3.2-cpu.onnx"),
|
||||
cpu_count=1, params=SmartTurnParams())
|
||||
analyzer.set_sample_rate(sr)
|
||||
frame_samples = sr * 32 // 1000
|
||||
speaking = False
|
||||
completed = False
|
||||
last_origin = None
|
||||
last_stop_ms = None
|
||||
last_action = None
|
||||
previous_vad_start = None
|
||||
pointer = 0
|
||||
results = []
|
||||
users = [r for r in history if r["role"] == "user"]
|
||||
agents = [r for r in history if r["role"] == "agent"]
|
||||
|
||||
def event(kind, result=None, origin=None):
|
||||
return dict(call_id=call_id, event_ms=round(clock.seconds*1000, 3),
|
||||
event=kind, probability=result.probability if result else None,
|
||||
prediction=int(result.is_complete) if result else None,
|
||||
audio_origin_ms=round(origin*1000, 3) if origin is not None else None)
|
||||
|
||||
for label in sorted(labels, key=lambda r: int(r["decision_ms"])):
|
||||
decision = int(label["decision_ms"])
|
||||
cutoff = decision * sr // 1000
|
||||
if cutoff > len(pcm):
|
||||
raise ValueError(f"Decision beyond audio: {call_id} {decision}")
|
||||
# Only complete transport frames observed by this checkpoint are fed.
|
||||
while pointer + frame_samples <= cutoff:
|
||||
end = pointer + frame_samples
|
||||
clock.seconds = end / sr
|
||||
chunk = pcm[pointer:end].tobytes()
|
||||
state = vad._run_analyzer(chunk)
|
||||
new_start = state == VADState.SPEAKING and not speaking
|
||||
new_stop = state == VADState.QUIET and speaking
|
||||
# The aggregator queues VAD events: raw audio reaches the turn
|
||||
# strategy with its old speech flag, followed by the VAD event.
|
||||
origin = source_audio_start(analyzer) if analyzer.speech_triggered else None
|
||||
appended = analyzer.append_audio(chunk, speaking)
|
||||
if analyzer.speech_triggered:
|
||||
last_origin = source_audio_start(analyzer)
|
||||
if appended == EndOfTurnState.COMPLETE:
|
||||
completed = True
|
||||
last_action = event("silence_timeout", origin=origin)
|
||||
events.append(last_action)
|
||||
if new_start:
|
||||
speaking = True
|
||||
completed = False
|
||||
last_action = None
|
||||
previous_vad_start = clock.seconds * 1000
|
||||
analyzer.update_vad_start_secs(vad.params.start_secs)
|
||||
events.append(event("vad_start"))
|
||||
elif new_stop:
|
||||
speaking = False
|
||||
last_stop_ms = clock.seconds * 1000
|
||||
origin = source_audio_start(analyzer)
|
||||
state, result = analyzer._process_speech_segment(analyzer._audio_buffer)
|
||||
last_origin = origin
|
||||
events.append(event("vad_stop", result, origin))
|
||||
if state == EndOfTurnState.COMPLETE:
|
||||
completed = True
|
||||
last_action = event("model_complete", result, origin)
|
||||
events.append(last_action)
|
||||
analyzer.clear()
|
||||
else:
|
||||
completed = False
|
||||
pointer = end
|
||||
|
||||
# Non-mutating model probe at EXACT decision_ms, using the start
|
||||
# selected by the current cache; after an earlier completion, use the
|
||||
# last completed segment start and label that diagnostic explicitly.
|
||||
active = analyzer.speech_triggered
|
||||
origin = source_audio_start(analyzer) if active else last_origin
|
||||
source = "active_turn" if active else "previous_completed_turn"
|
||||
prob = None
|
||||
prediction = None
|
||||
start_sample = None
|
||||
real_ms = None
|
||||
if origin is not None:
|
||||
start_sample = max(round(origin*sr), cutoff - int(sr*8))
|
||||
if start_sample < cutoff:
|
||||
sample = pcm[start_sample:cutoff].astype(np.float32) / 32768.0
|
||||
output = analyzer._predict_endpoint(sample)
|
||||
prob, prediction = output["probability"], output["prediction"]
|
||||
real_ms = len(sample)*1000/sr
|
||||
assert start_sample <= cutoff and real_ms <= 8000
|
||||
|
||||
before_users = [r for r in users if int(r["start_ms"]) < decision]
|
||||
current_user = max(before_users, key=lambda r: int(r["start_ms"])) if before_users else None
|
||||
# Incomplete ASR history does not invalidate an observed audio/VAD
|
||||
# prediction; keep it as a separate alignment issue for review.
|
||||
input_valid = prediction is not None
|
||||
current_asr_speaking = any(int(r["start_ms"]) <= decision < int(r["end_ms"]) for r in users)
|
||||
current_agent_speaking = any(int(r["start_ms"]) < decision < int(r["end_ms"]) for r in agents)
|
||||
label_value = int(label["can_reply_now"]) if label["can_reply_now"] in ("0", "1") else None
|
||||
strategy_prediction = int(completed)
|
||||
result = dict(label)
|
||||
result.update(input_valid=int(input_valid),
|
||||
input_issue="" if input_valid else "no_vad_origin",
|
||||
history_issue="" if current_user is not None else "no_prior_user_asr_segment",
|
||||
source_audio_channel=channel,
|
||||
point_probability=prob, point_prediction=prediction,
|
||||
point_correct=int(prediction == label_value) if input_valid else None,
|
||||
point_audio_source=source,
|
||||
point_audio_start_ms=start_sample*1000/sr if start_sample is not None else None,
|
||||
point_audio_end_ms=decision, point_real_audio_ms=real_ms,
|
||||
model_input_ms=8000, leading_zero_ms=8000-real_ms if real_ms is not None else None,
|
||||
vad_speaking=int(speaking), vad_last_start_ms=previous_vad_start,
|
||||
vad_last_stop_ms=last_stop_ms,
|
||||
strategy_prediction=strategy_prediction,
|
||||
strategy_correct=int(strategy_prediction == label_value) if input_valid else None,
|
||||
strategy_last_action_ms=last_action["event_ms"] if last_action else None,
|
||||
strategy_last_action=last_action["event"] if last_action else None,
|
||||
strategy_probability=last_action["probability"] if last_action else None,
|
||||
asr_user_segment_crosses_point=int(current_asr_speaking),
|
||||
asr_agent_segment_crosses_point=int(current_agent_speaking),
|
||||
label_status="draft" if label["annotator"] == "xlsx_draft" else "unconfirmed_manual")
|
||||
results.append(result)
|
||||
vad._executor.shutdown(wait=True)
|
||||
return results, dict(call_id=call_id, sample_rate=sr, channels=audio.shape[1],
|
||||
selected_user_channel=channel, audio_sha256=digest(audio_path))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--channel", type=int, default=0)
|
||||
parser.add_argument("--output", type=Path, default=HERE / "results" / "work-data-pipecat")
|
||||
args = parser.parse_args()
|
||||
labels_path = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
||||
history_path = ROOT / "labels/EOT标注/工作数据/asr_history.csv"
|
||||
labels, history = read_csv(labels_path), read_csv(history_path)
|
||||
keys = [(r["call_id"], r["decision_ms"]) for r in labels]
|
||||
assert len(set(keys)) == len(keys), "Duplicate annotation keys"
|
||||
clock = MediaClock()
|
||||
base_module.time = SimpleNamespace(monotonic=clock.monotonic, perf_counter=time.perf_counter)
|
||||
silero_module.time = SimpleNamespace(time=clock.time)
|
||||
logger.remove()
|
||||
logger.add(lambda message: None, level="ERROR")
|
||||
events, predictions, audio_inputs = [], [], []
|
||||
for call_id in sorted({r["call_id"] for r in labels}):
|
||||
clock.seconds = 0
|
||||
rows, meta = replay(call_id, [r for r in labels if r["call_id"] == call_id],
|
||||
[r for r in history if r["call_id"] == call_id],
|
||||
ROOT / "data/韩泰机器人抢话/音频" / f"{call_id}.mp3",
|
||||
clock, events, args.channel)
|
||||
predictions.extend(rows)
|
||||
audio_inputs.append(meta)
|
||||
print(f"{call_id}: {len(rows)} points; {sum(r['input_valid'] for r in rows)} eligible", flush=True)
|
||||
predictions.sort(key=lambda r: keys.index((r["call_id"], r["decision_ms"])))
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
write_csv(args.output / "predictions.csv", predictions)
|
||||
write_csv(args.output / "events.csv", events)
|
||||
errors = [r for r in predictions if r["input_valid"] and
|
||||
(not r["point_correct"] or not r["strategy_correct"])]
|
||||
write_csv(args.output / "errors.csv", errors)
|
||||
grouped = {}
|
||||
for field in ("label_status", "call_id", "reason", "point_audio_source"):
|
||||
grouped[field] = {value: {
|
||||
"point_model": metrics([r for r in predictions if r[field] == value], "point_prediction"),
|
||||
"audio_strategy": metrics([r for r in predictions if r[field] == value], "strategy_prediction")}
|
||||
for value in sorted({r[field] for r in predictions})}
|
||||
model_path = HERE / "smart-turn-v3.2-cpu.onnx"
|
||||
summary = dict(
|
||||
total_points=len(predictions), eligible_points=sum(r["input_valid"] for r in predictions),
|
||||
official_verified_points=0,
|
||||
point_model=metrics(predictions, "point_prediction"),
|
||||
audio_strategy=metrics(predictions, "strategy_prediction"),
|
||||
grouped=grouped,
|
||||
alignment_issue_points=sum(bool(r["history_issue"]) for r in predictions),
|
||||
history_aligned_subset={
|
||||
"point_model": metrics([r for r in predictions if not r["history_issue"]], "point_prediction"),
|
||||
"audio_strategy": metrics([r for r in predictions if not r["history_issue"]], "strategy_prediction")},
|
||||
config=dict(pipecat_version=importlib.metadata.version("pipecat-ai"),
|
||||
onnxruntime_version=importlib.metadata.version("onnxruntime"),
|
||||
vad_params=VADParams().model_dump(), smart_turn_params=SmartTurnParams().model_dump(),
|
||||
frame_ms=32, media_clock=True, cpu_count=1, threshold=0.5,
|
||||
model_sha256=digest(model_path), label_sha256=digest(labels_path),
|
||||
history_sha256=digest(history_path), audio_inputs=audio_inputs,
|
||||
replay_source_sha256={str(cls.__module__): digest(Path(inspect.getfile(cls)))
|
||||
for cls in (SileroVADAnalyzer, LocalSmartTurnAnalyzerV3, base_module.BaseSmartTurn)}),
|
||||
definitions={
|
||||
"point_model": "Non-mutating diagnostic inference at exact decision_ms. Active cache origin is used; after replay completed a turn, last completed origin is used and explicitly flagged. These extra queries never change replay state.",
|
||||
"audio_strategy": "Whether audio-only EOT is latched at decision_ms: actual VAD stops trigger inference; resumed VAD speech clears the latch; incomplete preserves cache; complete clears cache; silence timeout forces completion. It does not measure whether an earlier wrong EOT happened before speech resumed.",
|
||||
"event_order": "Audio appended with previous speaking flag, then queued VAD state changes handled. 32ms frames; no future or partial frames fed to VAD before a checkpoint.",
|
||||
"silero_reset": "Pipecat periodic five-second Silero state resets use media clock, not processing wall time.",
|
||||
"asr": "No realtime ASR return/finalization logs; transcript wait gate, external 5s controller timeout and ASR fallback are not simulated. History only audits point validity and speaker overlap, never enters model input.",
|
||||
"labels": "29 xlsx_draft and 15 no annotator/confidence; exploratory results only, no confirmed-label production accuracy claim.",
|
||||
"channel": "User channel 0 selected by RMS contrasts on non-overlapping ASR user/agent segments; echo/overlap can remain.",
|
||||
"invalid_points": "No detected VAD origin/prediction: excluded from metrics. Missing prior ASR user segments are alignment issues, retained in exploratory audio metrics and separately reported.",
|
||||
"scope": "Selected point classification/state snapshots; no complete turn boundaries, interval-level interruption rates or end latency claims."})
|
||||
(args.output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2)+"\n")
|
||||
print(json.dumps({k: summary[k] for k in ("total_points", "eligible_points", "point_model", "audio_strategy")}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
from transformers import WhisperFeatureExtractor
|
||||
|
||||
from audio_utils import truncate_audio_to_last_n_seconds
|
||||
|
||||
ONNX_MODEL_PATH = str(Path(__file__).with_name("smart-turn-v3.2-cpu.onnx"))
|
||||
|
||||
def build_session(onnx_path):
|
||||
so = ort.SessionOptions()
|
||||
so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
|
||||
so.inter_op_num_threads = 1
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
return ort.InferenceSession(onnx_path, sess_options=so, providers=["CPUExecutionProvider"])
|
||||
|
||||
feature_extractor = WhisperFeatureExtractor(chunk_length=8)
|
||||
session = build_session(ONNX_MODEL_PATH)
|
||||
|
||||
|
||||
def predict_endpoint(audio_array):
|
||||
"""
|
||||
Predict whether an audio segment is complete (turn ended) or incomplete.
|
||||
|
||||
Args:
|
||||
audio_array: Numpy array containing audio samples at 16kHz
|
||||
|
||||
Returns:
|
||||
Dictionary containing prediction results:
|
||||
- prediction: 1 for complete, 0 for incomplete
|
||||
- probability: Probability of completion (sigmoid output)
|
||||
"""
|
||||
|
||||
# Truncate to 8 seconds (keeping the end) or pad to 8 seconds
|
||||
audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8)
|
||||
|
||||
# Process audio using Whisper's feature extractor
|
||||
inputs = feature_extractor(
|
||||
audio_array,
|
||||
sampling_rate=16000,
|
||||
return_tensors="np",
|
||||
padding="max_length",
|
||||
max_length=8 * 16000,
|
||||
truncation=True,
|
||||
do_normalize=True,
|
||||
)
|
||||
|
||||
# Extract features and ensure correct shape for ONNX
|
||||
input_features = inputs.input_features.squeeze(0).astype(np.float32)
|
||||
input_features = np.expand_dims(input_features, axis=0) # Add batch dimension
|
||||
|
||||
# Run ONNX inference
|
||||
outputs = session.run(None, {"input_features": input_features})
|
||||
|
||||
# Extract probability (ONNX model returns sigmoid probabilities)
|
||||
probability = outputs[0][0].item()
|
||||
|
||||
# Make prediction (1 for Complete, 0 for Incomplete)
|
||||
prediction = 1 if probability > 0.5 else 0
|
||||
|
||||
return {
|
||||
"prediction": prediction,
|
||||
"probability": probability,
|
||||
}
|
||||
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
# Create a dummy audio array for testing (1 second of random audio)
|
||||
dummy_audio = np.random.randn(16000).astype(np.float32)
|
||||
|
||||
result = predict_endpoint(dummy_audio)
|
||||
print(f"Prediction: {result['prediction']}")
|
||||
print(f"Probability: {result['probability']:.4f}")
|
||||
@@ -0,0 +1,61 @@
|
||||
import argparse
|
||||
import librosa
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
from inference import predict_endpoint
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Smart Turn v3.2 CPU inference")
|
||||
parser.add_argument("audio", help="Audio file to analyze")
|
||||
parser.add_argument("--start-ms", type=int, default=0, help="Current user turn start")
|
||||
parser.add_argument("--end-ms", type=int, help="Decision time (exclusive audio cutoff)")
|
||||
args = parser.parse_args()
|
||||
if args.start_ms < 0 or (args.end_ms is not None and args.end_ms <= args.start_ms):
|
||||
parser.error("Require 0 <= start-ms < end-ms")
|
||||
file_path = args.audio
|
||||
|
||||
try:
|
||||
print(f"Loading audio file: {file_path}")
|
||||
# Load the audio file with original sample rate
|
||||
audio, sr = librosa.load(file_path, sr=None, mono=True)
|
||||
|
||||
# Slice before resampling so no audio after the decision is included.
|
||||
start_sample = args.start_ms * sr // 1000
|
||||
end_sample = len(audio) if args.end_ms is None else args.end_ms * sr // 1000
|
||||
if start_sample >= len(audio) or end_sample > len(audio):
|
||||
raise ValueError("Requested time range lies outside the audio")
|
||||
audio = audio[start_sample:end_sample]
|
||||
|
||||
print(f"Loaded audio with sample rate: {sr} Hz, duration: {len(audio) / sr:.2f} seconds")
|
||||
|
||||
# If needed, resample to 16kHz (the model's expected sample rate)
|
||||
if sr != 16000:
|
||||
print(f"Resampling from {sr}Hz to 16000Hz")
|
||||
audio = librosa.resample(audio, orig_sr=sr, target_sr=16000)
|
||||
|
||||
# Convert audio to float32 if not already
|
||||
if audio.dtype != np.float32:
|
||||
audio = audio.astype(np.float32)
|
||||
|
||||
# Make sure the audio is in the expected range [-1, 1]
|
||||
if np.max(np.abs(audio)) > 1.0:
|
||||
audio = audio / np.max(np.abs(audio))
|
||||
|
||||
# Call the prediction function with both audio and sample rate
|
||||
print("Running endpoint prediction...")
|
||||
result = predict_endpoint(audio)
|
||||
|
||||
# Display results
|
||||
print("\nResults:")
|
||||
print(f"Prediction: {'Complete' if result['prediction'] == 1 else 'Incomplete'}")
|
||||
print(f"Probability of complete: {result['probability']:.4f}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing audio file: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
[project]
|
||||
name = "smart-turn-v3-2"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"numpy",
|
||||
"onnxruntime",
|
||||
"librosa",
|
||||
"soundfile",
|
||||
"transformers==4.48.2",
|
||||
"pipecat-ai>=1.12.0",
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "tuna"
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
default = true
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,probability,prediction,correct,error_type,audio_start_ms,audio_end_ms,real_audio_ms,model_input_ms,leading_zero_ms,user_block_origin_ms,input_source,alignment_issue,audio_channel,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,0.9848582744598389,1,0,false_reply,40640.0,48125,7485.0,8000,515.0,40640,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,0.9729918241500854,1,0,false_reply,69860.0,76672,6812.0,8000,1188.0,69860,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,轮胎裂纹。,0,low,mid_turn_pause,xlsx_draft,0.9777120351791382,1,0,false_reply,62000.0,65298,3298.0,8000,4702.0,62000,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,轮胎裂纹同意更换新轮胎。但是,0,low,mid_turn_pause,xlsx_draft,0.6701734066009521,1,0,false_reply,62000.0,67943,5943.0,8000,2057.0,62000,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,0.8732463121414185,1,0,false_reply,62430.0,70430,8000.0,8000,0.0,62000,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,迈腾2023款330豪华版2.0T。出现的问题是轮购车时间是2024年5月份，里程是6851公里。,0,low,unfinished_expression,xlsx_draft,0.9333261251449585,1,0,false_reply,43080.0,51080,8000.0,8000,0.0,33980,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,0.9889053702354431,1,0,false_reply,60320.0,68313,7993.0,8000,7.0,60320,asr_user_block,,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70745,轮胎开裂,0,low,mid_turn_pause,xlsx_draft,0.9604700207710266,1,0,false_reply,68160.0,70745,2585.0,8000,5415.0,68160,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,40701,帕萨特,0,low,mid_turn_pause,xlsx_draft,0.9875311851501465,1,0,false_reply,37980.0,40701,2721.0,8000,5279.0,37980,asr_user_block,,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,0.978554368019104,1,0,false_reply,41620.0,47187,5567.0,8000,2433.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,52326,我的轮胎开裂了。,0,,mid_turn_pause,,0.9859892129898071,1,0,false_reply,44326.0,52326,8000.0,8000,0.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48348,爱车型号是大众帕萨特，购车时间2023年10月份,0,,mid_turn_pause,,0.9889053702354431,1,0,false_reply,41260.0,48348,7088.0,8000,912.0,41260,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,44810,帕萨特,0,,mid_turn_pause,,0.9777120351791382,1,0,false_reply,41620.0,44810,3190.0,8000,4810.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,45581,帕萨特,0,,mid_turn_pause,,0.7923524379730225,1,0,false_reply,41620.0,45581,3961.0,8000,4039.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,0.9870367050170898,1,0,false_reply,33980.0,41219,7239.0,8000,761.0,33980,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41207,我是2023年,0,,mid_turn_pause,,0.9403132200241089,1,0,false_reply,34480.0,41207,6727.0,8000,1273.0,34480,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,0.9808988571166992,1,0,false_reply,41204.0,49204,8000.0,8000,0.0,34480,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73640,轮胎开裂，行驶28000公里，轮胎开裂,0,,mid_turn_pause,,0.9870367050170898,1,0,false_reply,68160.0,73640,5480.0,8000,2520.0,68160,asr_user_block,,0,unconfirmed_manual
|
||||
|
@@ -0,0 +1,45 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,probability,prediction,correct,error_type,audio_start_ms,audio_end_ms,real_audio_ms,model_input_ms,leading_zero_ms,user_block_origin_ms,input_source,alignment_issue,audio_channel,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,21780,贵州省遵义市仁怀市。,1,low,completed_answer,xlsx_draft,0.9875311851501465,1,1,correct,15980.0,21780,5800.0,8000,2200.0,15980,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,29860,投诉建议。,1,low,completed_answer,xlsx_draft,0.9403132200241089,1,1,correct,26000.0,29860,3860.0,8000,4140.0,26000,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,0.9848582744598389,1,0,false_reply,40640.0,48125,7485.0,8000,515.0,40640,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,64000,行驶3万公里。,1,low,completed_answer,xlsx_draft,0.9875311851501465,1,1,correct,60200.0,64000,3800.0,8000,4200.0,60200,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,0.9729918241500854,1,0,false_reply,69860.0,76672,6812.0,8000,1188.0,69860,asr_user_block,,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,尽快，已经影响行车安全了。,1,low,completed_answer,xlsx_draft,0.9729918241500854,1,1,correct,84780.0,88482,3702.0,8000,4298.0,84780,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,22007,宣传。,1,low,completed_answer,xlsx_draft,0.9865230321884155,1,1,correct,18720.0,22007,3287.0,8000,4713.0,18720,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,31842,投诉建议。,1,low,completed_answer,xlsx_draft,0.9889053702354431,1,1,correct,28540.0,31842,3302.0,8000,4698.0,28540,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,52348,爱车型号是大众帕萨特，购车时间2023年10月份，行驶里程24000多公里。,1,low,completed_answer,xlsx_draft,0.9889053702354431,1,1,correct,44348.0,52348,8000.0,8000,0.0,41260,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,轮胎裂纹。,0,low,mid_turn_pause,xlsx_draft,0.9777120351791382,1,0,false_reply,62000.0,65298,3298.0,8000,4702.0,62000,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,轮胎裂纹同意更换新轮胎。但是,0,low,mid_turn_pause,xlsx_draft,0.6701734066009521,1,0,false_reply,62000.0,67943,5943.0,8000,2057.0,62000,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,0.8732463121414185,1,0,false_reply,62430.0,70430,8000.0,8000,0.0,62000,asr_user_block,,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,82941,可以。,1,low,completed_answer,xlsx_draft,0.9801466464996338,1,1,correct,80060.0,82941,2881.0,8000,5119.0,80060,asr_user_block,,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,17153,我姓宋。,1,low,completed_answer,xlsx_draft,0.9880069494247437,1,1,correct,14100.0,17153,3053.0,8000,4947.0,14100,asr_user_block,,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,24846,郑州市，河南郑州市。,1,low,completed_answer,xlsx_draft,0.9884648323059082,1,1,correct,19720.0,24846,5126.0,8000,2874.0,19720,asr_user_block,,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,33471,我的轮胎开裂了。,1,low,completed_answer,xlsx_draft,0.9854345321655273,1,1,correct,29780.0,33471,3691.0,8000,4309.0,29780,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,18027,天津市。,1,low,completed_answer,xlsx_draft,0.9884648323059082,1,1,correct,14760.0,18027,3267.0,8000,4733.0,14760,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,24768,投诉。,1,low,completed_answer,xlsx_draft,0.9880069494247437,1,1,correct,22060.0,24768,2708.0,8000,5292.0,22060,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,迈腾2023款330豪华版2.0T。出现的问题是轮购车时间是2024年5月份，里程是6851公里。,0,low,unfinished_expression,xlsx_draft,0.9333261251449585,1,0,false_reply,43080.0,51080,8000.0,8000,0.0,33980,asr_user_block,,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,0.9889053702354431,1,0,false_reply,60320.0,68313,7993.0,8000,7.0,60320,asr_user_block,,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,13540,辽宁葫芦岛。,1,low,completed_answer,xlsx_draft,0.9793655872344971,1,1,correct,10244.0,13540,3296.0,8000,4704.0,10244.0,past_vad_missing_asr,missing_earlier_user_asr_segment,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,27520,投诉建议。,1,low,completed_answer,xlsx_draft,0.9870367050170898,1,1,correct,23920.0,27520,3600.0,8000,4400.0,23920,asr_user_block,,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,60790,我的轮胎行驶28000公里。,0,low,unfinished_expression,xlsx_draft,0.04772377014160156,0,1,correct,55780.0,60790,5010.0,8000,2990.0,55780,asr_user_block,,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70745,轮胎开裂,0,low,mid_turn_pause,xlsx_draft,0.9604700207710266,1,0,false_reply,68160.0,70745,2585.0,8000,5415.0,68160,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,18842,苏州昆山。,1,low,completed_answer,xlsx_draft,0.8732463121414185,1,1,correct,14560.0,18842,4282.0,8000,3718.0,14560,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,29619,投诉。,1,low,completed_answer,xlsx_draft,0.9604700207710266,1,1,correct,26560.0,29619,3059.0,8000,4941.0,26560,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,40701,帕萨特,0,low,mid_turn_pause,xlsx_draft,0.9875311851501465,1,0,false_reply,37980.0,40701,2721.0,8000,5279.0,37980,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,54600,24年11月份购买的帕萨特，15000公里左右。,1,low,completed_answer,xlsx_draft,0.9170267581939697,1,1,correct,46600.0,54600,8000.0,8000,0.0,37980,asr_user_block,,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,65819,四条轮胎，最近我在。,0,low,unfinished_expression,xlsx_draft,0.01514175534248352,0,1,correct,60380.0,65819,5439.0,8000,2561.0,60380,asr_user_block,,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,0.978554368019104,1,0,false_reply,41620.0,47187,5567.0,8000,2433.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,52326,我的轮胎开裂了。,0,,mid_turn_pause,,0.9859892129898071,1,0,false_reply,44326.0,52326,8000.0,8000,0.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48348,爱车型号是大众帕萨特，购车时间2023年10月份,0,,mid_turn_pause,,0.9889053702354431,1,0,false_reply,41260.0,48348,7088.0,8000,912.0,41260,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,44810,帕萨特,0,,mid_turn_pause,,0.9777120351791382,1,0,false_reply,41620.0,44810,3190.0,8000,4810.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,49500,帕萨特L2025款2025年3月购车,1,,completed_turn,,0.9842595458030701,1,1,correct,41620.0,49500,7880.0,8000,120.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,54318,行驶里程29000公里。,1,,completed_turn,,0.9859892129898071,1,1,correct,46318.0,54318,8000.0,8000,0.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,45581,帕萨特,0,,mid_turn_pause,,0.7923524379730225,1,0,false_reply,41620.0,45581,3961.0,8000,4039.0,41620,asr_user_block,,0,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,0.9870367050170898,1,0,false_reply,33980.0,41219,7239.0,8000,761.0,33980,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,19620,,1,,completed_turn,,0.9870367050170898,1,1,correct,16100.0,19620,3520.0,8000,4480.0,16100,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41207,我是2023年,0,,mid_turn_pause,,0.9403132200241089,1,0,false_reply,34480.0,41207,6727.0,8000,1273.0,34480,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,0.9808988571166992,1,0,false_reply,41204.0,49204,8000.0,8000,0.0,34480,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,58424,我的轮胎,0,,mid_turn_pause,,0.044268637895584106,0,1,correct,55780.0,58424,2644.0,8000,5356.0,55780,asr_user_block,,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73640,轮胎开裂，行驶28000公里，轮胎开裂,0,,mid_turn_pause,,0.9870367050170898,1,0,false_reply,68160.0,73640,5480.0,8000,2520.0,68160,asr_user_block,,0,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,11531,姓赵,1,,completed_turn,,0.9880069494247437,1,1,correct,8740.0,11531,2791.0,8000,5209.0,8740.0,past_vad_missing_asr,missing_earlier_user_asr_segment,0,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,62828,四条轮胎,0,,mid_turn_pause,,0.07174640893936157,0,1,correct,60380.0,62828,2448.0,8000,5552.0,60380,asr_user_block,,0,unconfirmed_manual
|
||||
|
@@ -0,0 +1,329 @@
|
||||
{
|
||||
"total_points": 44,
|
||||
"metrics": {
|
||||
"n": 44,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.55,
|
||||
"f1": 0.7096774193548387
|
||||
},
|
||||
"grouped": {
|
||||
"label_status": {
|
||||
"draft": {
|
||||
"n": 29,
|
||||
"tn": 2,
|
||||
"fp": 9,
|
||||
"fn": 0,
|
||||
"tp": 18,
|
||||
"accuracy": 0.6896551724137931,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.8
|
||||
},
|
||||
"unconfirmed_manual": {
|
||||
"n": 15,
|
||||
"tn": 2,
|
||||
"fp": 9,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.3076923076923077,
|
||||
"f1": 0.47058823529411764
|
||||
}
|
||||
},
|
||||
"call_id": {
|
||||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||||
"n": 6,
|
||||
"tn": 0,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.6666666666666666,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.8
|
||||
},
|
||||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||||
"n": 8,
|
||||
"tn": 0,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5,
|
||||
"f1": 0.6666666666666666
|
||||
},
|
||||
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||||
"n": 9,
|
||||
"tn": 0,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 5,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5555555555555556,
|
||||
"f1": 0.7142857142857143
|
||||
},
|
||||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||||
"n": 5,
|
||||
"tn": 0,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 2,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.4,
|
||||
"f1": 0.5714285714285714
|
||||
},
|
||||
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||||
"n": 9,
|
||||
"tn": 2,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 3,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 0.6666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.42857142857142855,
|
||||
"f1": 0.6
|
||||
},
|
||||
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||||
"n": 7,
|
||||
"tn": 2,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.8571428571428571,
|
||||
"false_reply_rate": 0.3333333333333333,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.8,
|
||||
"f1": 0.8888888888888888
|
||||
}
|
||||
},
|
||||
"reason": {
|
||||
"completed_answer": {
|
||||
"n": 18,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 18,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
},
|
||||
"completed_turn": {
|
||||
"n": 4,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
},
|
||||
"mid_turn_pause": {
|
||||
"n": 17,
|
||||
"tn": 2,
|
||||
"fp": 15,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.11764705882352941,
|
||||
"false_reply_rate": 0.8823529411764706,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
},
|
||||
"unfinished_expression": {
|
||||
"n": 5,
|
||||
"tn": 2,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 0.6,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
}
|
||||
},
|
||||
"input_source": {
|
||||
"asr_user_block": {
|
||||
"n": 42,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 20,
|
||||
"accuracy": 0.5714285714285714,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5263157894736842,
|
||||
"f1": 0.6896551724137931
|
||||
},
|
||||
"past_vad_missing_asr": {
|
||||
"n": 2,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 2,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"pipecat_version": "1.12.0",
|
||||
"model": "smart-turn-v3.2-cpu.onnx",
|
||||
"threshold": 0.5,
|
||||
"pre_speech_ms": 500,
|
||||
"max_duration_secs": 8,
|
||||
"continuation_gap_ms": 3000,
|
||||
"selected_audio_channel": 0,
|
||||
"model_sha256": "2bb026316b14a660486a75b1733cd3fbab8c2fd0314dc9af7be49f8cca967e4f",
|
||||
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||||
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa"
|
||||
},
|
||||
"method": "Each exact decision_ms is classified directly by the model. Earlier model scores, EOT signals, VAD stop gates, silence timeout forced replies and ASR return gates never determine prediction or clear this input.",
|
||||
"threshold_diagnostics": [
|
||||
{
|
||||
"threshold": 0.5,
|
||||
"n": 44,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.55,
|
||||
"f1": 0.7096774193548387
|
||||
},
|
||||
{
|
||||
"threshold": 0.7,
|
||||
"n": 44,
|
||||
"tn": 5,
|
||||
"fp": 17,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.7727272727272727,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5641025641025641,
|
||||
"f1": 0.7213114754098361
|
||||
},
|
||||
{
|
||||
"threshold": 0.8,
|
||||
"n": 44,
|
||||
"tn": 6,
|
||||
"fp": 16,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.7272727272727273,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5789473684210527,
|
||||
"f1": 0.7333333333333333
|
||||
},
|
||||
{
|
||||
"threshold": 0.9,
|
||||
"n": 44,
|
||||
"tn": 7,
|
||||
"fp": 15,
|
||||
"fn": 1,
|
||||
"tp": 21,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.6818181818181818,
|
||||
"false_hold_rate": 0.045454545454545456,
|
||||
"recall": 0.9545454545454546,
|
||||
"precision": 0.5833333333333334,
|
||||
"f1": 0.7241379310344828
|
||||
},
|
||||
{
|
||||
"threshold": 0.95,
|
||||
"n": 44,
|
||||
"tn": 9,
|
||||
"fp": 13,
|
||||
"fn": 3,
|
||||
"tp": 19,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.5909090909090909,
|
||||
"false_hold_rate": 0.13636363636363635,
|
||||
"recall": 0.8636363636363636,
|
||||
"precision": 0.59375,
|
||||
"f1": 0.7037037037037037
|
||||
},
|
||||
{
|
||||
"threshold": 0.98,
|
||||
"n": 44,
|
||||
"tn": 14,
|
||||
"fp": 8,
|
||||
"fn": 6,
|
||||
"tp": 16,
|
||||
"accuracy": 0.6818181818181818,
|
||||
"false_reply_rate": 0.36363636363636365,
|
||||
"false_hold_rate": 0.2727272727272727,
|
||||
"recall": 0.7272727272727273,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.6956521739130435
|
||||
},
|
||||
{
|
||||
"threshold": 0.99,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
}
|
||||
],
|
||||
"audio_origin": "Past ASR user segments locate current speech block. Consecutive user segments within 3 s share the first start unless a complete non-overlapping robot question separates them. Keep 500 ms before that start and then last 8 s. Intra-segment pauses are not resolved by coarse ASR timing. Two missing-ASR points use only past Silero VAD starts; full-audio model events are not used.",
|
||||
"limitations": [
|
||||
"Exploratory scoring: 29 xlsx_draft and 15 rows missing annotator/confidence; no explicit verified status.",
|
||||
"Coarse ASR speech block origin approximates the discussed cache rule; this is point classification, not a complete online Pipecat strategy replay.",
|
||||
"cb3238dc 13540 text is 辽宁葫芦岛 but its corresponding user ASR starts at 16600; review alignment. e304f740 11531 surname utterance is absent from ASR history.",
|
||||
"User channel 0 can contain robot echo/overlap; no future ASR-based masking.",
|
||||
"Selected-point false-reply rate is not interval-level interruption rate."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
threshold,n,tn,fp,fn,tp,accuracy,false_reply_rate,false_hold_rate,recall,precision,f1
|
||||
0.5,44,4,18,0,22,0.5909090909090909,0.8181818181818182,0.0,1.0,0.55,0.7096774193548387
|
||||
0.7,44,5,17,0,22,0.6136363636363636,0.7727272727272727,0.0,1.0,0.5641025641025641,0.7213114754098361
|
||||
0.8,44,6,16,0,22,0.6363636363636364,0.7272727272727273,0.0,1.0,0.5789473684210527,0.7333333333333333
|
||||
0.9,44,7,15,1,21,0.6363636363636364,0.6818181818181818,0.045454545454545456,0.9545454545454546,0.5833333333333334,0.7241379310344828
|
||||
0.95,44,9,13,3,19,0.6363636363636364,0.5909090909090909,0.13636363636363635,0.8636363636363636,0.59375,0.7037037037037037
|
||||
0.98,44,14,8,6,16,0.6818181818181818,0.36363636363636365,0.2727272727272727,0.7272727272727273,0.6666666666666666,0.6956521739130435
|
||||
0.99,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
|
@@ -0,0 +1,56 @@
|
||||
Smart Turn v3.2：工作数据 can_reply_now 判断点分类
|
||||
2026-10-12
|
||||
|
||||
本版仅评估 can_reply_now。每个 decision_ms 独立输入模型，使用 probability > 0.5 映射为可回复；不把已有结束信号、VAD 尚未停止、3 秒超时或 ASR 等待作为预测结果。
|
||||
|
||||
输入规则
|
||||
用户第 1 声道（索引 0），原始 8 kHz 音频按当前点裁剪，然后由 Pipecat 1.12.0 的实际 LocalSmartTurnAnalyzerV3 重采样至 16 kHz。
|
||||
依据该点之前开始的用户 ASR 片段定位发言块；3 秒内相邻用户片段归并，完整且不重叠的机器人提问会分开两个发言块。起点前保留 500 毫秒，最多保留末尾 8 秒，短音频在前面补零至 8 秒。
|
||||
不使用人工标签或先前模型判定来决定音频起点，也不因早先模型判 Complete 清空当前输入。转写仅定位时间，文本不进入模型。
|
||||
这是利用片段级 ASR 近似当前用户话轮起点的判断点分类；单个 ASR 片段内部的停顿未据此重新切分，不代表完整 Pipecat 在线策略回放。
|
||||
2 个缺少早期用户 ASR 的点使用该判断点以前的实际 Silero VAD 起始事件补充起点，并单独标记。
|
||||
|
||||
结果：44 个点，人工可回复/等待各 22 个
|
||||
可回复：22 个全部正确，误阻止 0 个。
|
||||
等待：4 个正确，误放行 18 个。
|
||||
准确率：26/44 = 59.09%。
|
||||
可回复检出率：22/22 = 100.00%。
|
||||
误放行率：18/22 = 81.82%。
|
||||
放行精确率：22/40 = 55.00%。
|
||||
|
||||
按原因分析
|
||||
mid_turn_pause：17 个点，误放行 15 个（88.24%）。
|
||||
unfinished_expression：5 个点，误放行 3 个（60.00%）。
|
||||
completed_answer / completed_turn：合计 22 个点，全被判为可回复。
|
||||
当前模型较容易放行，主要不能可靠拒绝话轮中间停顿。
|
||||
|
||||
代表性误放行（标注判断点，概率）
|
||||
872a1e67，44.810 秒，“帕萨特”，0.97771。
|
||||
2c83253a，48.125 秒，“型号是24年380大众帕萨特星空龙腾版。”，0.98486。
|
||||
a266b9ef，41.219 秒，“迈腾2023款330豪华版2.0T”，0.98704。
|
||||
40ae686c，67.943 秒，“轮胎裂纹同意更换新轮胎。但是”，0.67017。
|
||||
40ae686c，70.430 秒，“轮胎裂纹同意更换新轮胎。但是20”，0.87325。
|
||||
前三个例子提示局部表达结束分数与当前任务的可接话条件可能不一致，例如车型说完但其他信息尚未回答。该解释需要结合回听确认；此处不据转写断言语气变化。
|
||||
|
||||
同一批判断点上的阈值诊断（固定音频，不推演结束信号）
|
||||
阈值 0.50：误放行 18/22，误阻止 0/22，准确率 59.09%。
|
||||
阈值 0.80：误放行 16/22，误阻止 0/22，准确率 63.64%。
|
||||
阈值 0.95：误放行 13/22，误阻止 3/22，准确率 63.64%。
|
||||
阈值 0.98：误放行 8/22，误阻止 6/22，准确率 68.18%。
|
||||
阈值 0.99：误放行 0/22，误阻止 22/22，准确率 50.00%。
|
||||
提高阈值可以减少误放行，但正负例分数重叠明显，无法在本批数据上同时保持高检出率和低误放行率。阈值对照为同集探索，不是独立测试集选参结果。
|
||||
|
||||
标签与对齐问题
|
||||
29 个点仍为 xlsx_draft，其余 15 个未填标注员/置信度，没有明确回听确认状态。以上结果作为探索性比较，不作正式准确率验收。
|
||||
cb3238dc 的 13.540 秒标签文本为“辽宁葫芦岛”，对应用户转写在 16.600 秒开始，应回听确认判断点或文本。
|
||||
e304f740 的 11.531 秒“姓赵”在 asr_history 中缺少对应早期用户片段。
|
||||
原始人工标签未修改，两个点均保留预测并标记 alignment_issue。
|
||||
第 1 声道仍可能有机器人回声或重叠语音；这批选定点的误放行率不能直接当成静音区间误抢话率。
|
||||
|
||||
输出
|
||||
predictions.csv：全部 44 个点的标签、模型概率、can_reply 预测、正确性及音频起止/长度。
|
||||
errors.csv：18 个误放行点。
|
||||
thresholds.csv：7 个阈值的同集诊断结果。
|
||||
summary.json：统计、分组、输入规则、版本及 SHA256。
|
||||
|
||||
建议优先回听车型/购车时间/里程组合回答的负例，确认话轮起点和判断点，再用已确认、按通话划分的测试集检验阈值或结合对话任务上下文的方法。
|
||||
@@ -0,0 +1,20 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,input_valid,input_issue,history_issue,source_audio_channel,point_probability,point_prediction,point_correct,point_audio_source,point_audio_start_ms,point_audio_end_ms,point_real_audio_ms,model_input_ms,leading_zero_ms,vad_speaking,vad_last_start_ms,vad_last_stop_ms,strategy_prediction,strategy_correct,strategy_last_action_ms,strategy_last_action,strategy_probability,asr_user_segment_crosses_point,asr_agent_segment_crosses_point,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9865230321884155,1,0,previous_completed_turn,46336.0,48125,1789.0,8000,6211.0,0,46624.0,47168.0,1,0,47168.0,model_complete,0.978554368019104,1,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,1,,,0,0.9880069494247437,1,0,previous_completed_turn,74208.0,76672,2464.0,8000,5536.0,0,74752.0,75200.0,1,0,75200.0,model_complete,0.9445840716362,1,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,尽快，已经影响行车安全了。,1,low,completed_answer,xlsx_draft,1,,,0,0.9403132200241089,1,1,active_turn,86400.0,88482,2082.0,8000,5918.0,1,86752.0,86400.0,0,0,,,,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,轮胎裂纹。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9522762298583984,1,0,previous_completed_turn,62080.0,65298,3218.0,8000,4782.0,0,62752.0,63680.0,1,0,63680.0,model_complete,0.7987582683563232,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,轮胎裂纹同意更换新轮胎。但是,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.8498003482818604,1,0,previous_completed_turn,65088.0,67943,2855.0,8000,5145.0,0,65760.0,66848.0,1,0,66848.0,model_complete,0.7040206789970398,1,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,1,,,0,0.7722399234771729,1,0,active_turn,67840.0,70430,2590.0,8000,5410.0,0,68512.0,70176.0,0,1,,,,1,1,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,迈腾2023款330豪华版2.0T。出现的问题是轮购车时间是2024年5月份，里程是6851公里。,0,low,unfinished_expression,xlsx_draft,1,,,0,0.8498003482818604,1,0,active_turn,48384.0,51080,2696.0,8000,5304.0,1,49056.0,47840.0,0,1,,,,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9897372126579285,1,0,previous_completed_turn,64384.0,68313,3929.0,8000,4071.0,0,65056.0,66976.0,1,0,66976.0,model_complete,0.9884648323059082,1,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70745,轮胎开裂,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9673051834106445,1,0,previous_completed_turn,69504.0,70745,1241.0,8000,6759.0,0,69920.0,70368.0,1,0,70368.0,model_complete,0.9836374521255493,1,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,40701,帕萨特,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9875311851501465,1,0,previous_completed_turn,38208.0,40701,2493.0,8000,5507.0,0,38880.0,39488.0,1,0,39488.0,model_complete,0.9007586240768433,1,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,1,,,0,0.9842595458030701,1,0,previous_completed_turn,41760.0,47187,5427.0,8000,2573.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,52326,我的轮胎开裂了。,0,,mid_turn_pause,,1,,,0,0.9875311851501465,1,0,previous_completed_turn,50624.0,52326,1702.0,8000,6298.0,0,51296.0,51712.0,1,0,51712.0,model_complete,0.9870367050170898,1,0,unconfirmed_manual
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48348,爱车型号是大众帕萨特，购车时间2023年10月份,0,,mid_turn_pause,,1,,,0,0.9893293380737305,1,0,previous_completed_turn,46048.0,48348,2300.0,8000,5700.0,0,46464.0,47776.0,1,0,47776.0,model_complete,0.963354766368866,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,44810,帕萨特,0,,mid_turn_pause,,1,,,0,0.9793655872344971,1,0,previous_completed_turn,41760.0,44810,3050.0,8000,4950.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,45581,帕萨特,0,,mid_turn_pause,,1,,,0,0.5393059253692627,1,0,previous_completed_turn,41760.0,45581,3821.0,8000,4179.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,1,,,0,0.9884648323059082,1,0,previous_completed_turn,38304.0,41219,2915.0,8000,5085.0,0,38592.0,39328.0,1,0,39328.0,model_complete,0.9859892129898071,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41207,我是2023年,0,,mid_turn_pause,,1,,,0,0.8817123174667358,1,0,previous_completed_turn,38176.0,41207,3031.0,8000,4969.0,0,38848.0,40384.0,1,0,40384.0,model_complete,0.9228270053863525,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,1,,,0,0.9604700207710266,1,0,previous_completed_turn,45888.0,49204,3316.0,8000,4684.0,0,46560.0,47968.0,1,0,47968.0,model_complete,0.9697068929672241,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73640,轮胎开裂，行驶28000公里，轮胎开裂,0,,mid_turn_pause,,1,,,0,0.9893293380737305,1,0,previous_completed_turn,70720.0,73640,2920.0,8000,5080.0,0,71392.0,73088.0,1,0,73088.0,model_complete,0.9854345321655273,1,0,unconfirmed_manual
|
||||
|
@@ -0,0 +1,230 @@
|
||||
call_id,event_ms,event,probability,prediction,audio_origin_ms
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,16864.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,18176.0,vad_stop,0.9540347456932068,1,16192.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,18176.0,model_complete,0.9540347456932068,1,16192.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,19168.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,19776.0,vad_stop,0.9793655872344971,1,18496.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,19776.0,model_complete,0.9793655872344971,1,18496.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,26880.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,27648.0,vad_stop,0.8971817493438721,1,26208.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,27648.0,model_complete,0.8971817493438721,1,26208.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,41376.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,42304.0,vad_stop,0.8050150871276855,1,40704.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,42304.0,model_complete,0.8050150871276855,1,40704.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,44832.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,45440.0,vad_stop,0.7652381658554077,1,44160.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,45440.0,model_complete,0.7652381658554077,1,44160.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,45760.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,46336.0,vad_stop,0.9540347456932068,1,45440.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,46336.0,model_complete,0.9540347456932068,1,45440.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,46624.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,47168.0,vad_stop,0.978554368019104,1,46336.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,47168.0,model_complete,0.978554368019104,1,46336.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48640.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,49216.0,vad_stop,0.9589470624923706,1,47968.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,49216.0,model_complete,0.9589470624923706,1,47968.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,55104.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,55520.0,vad_stop,0.9801466464996338,1,54432.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,55520.0,model_complete,0.9801466464996338,1,54432.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,58336.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,59008.0,vad_stop,0.9816229343414307,1,57664.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,59008.0,model_complete,0.9816229343414307,1,57664.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,70688.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,71584.0,vad_stop,0.7040206789970398,1,70016.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,71584.0,model_complete,0.7040206789970398,1,70016.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,71808.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,72320.0,vad_stop,0.8857588768005371,1,71584.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,72320.0,model_complete,0.8857588768005371,1,71584.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,73408.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,74208.0,vad_stop,0.9380639791488647,1,72736.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,74208.0,model_complete,0.9380639791488647,1,72736.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,74752.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,75200.0,vad_stop,0.9445840716362,1,74208.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,75200.0,model_complete,0.9445840716362,1,74208.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,77216.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,77984.0,vad_stop,0.9424859285354614,1,76544.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,77984.0,model_complete,0.9424859285354614,1,76544.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,78816.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,79008.0,vad_stop,0.9685282111167908,1,78144.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,79008.0,model_complete,0.9685282111167908,1,78144.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,85664.0,vad_start,,,
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,86400.0,vad_stop,0.9836374521255493,1,84992.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,86400.0,model_complete,0.9836374521255493,1,84992.0
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,86752.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,11136.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,11488.0,vad_stop,0.963354766368866,1,10464.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,11488.0,model_complete,0.963354766368866,1,10464.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,19488.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,19872.0,vad_stop,0.9333261251449585,1,18816.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,19872.0,model_complete,0.9333261251449585,1,18816.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,29312.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,29824.0,vad_stop,0.9801466464996338,1,28640.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,29824.0,model_complete,0.9801466464996338,1,28640.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,42016.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,43008.0,vad_stop,0.6252830028533936,1,41344.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,43008.0,model_complete,0.6252830028533936,1,41344.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,43744.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,44608.0,vad_stop,0.9540347456932068,1,43072.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,44608.0,model_complete,0.9540347456932068,1,43072.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,45472.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,46048.0,vad_stop,0.7040206789970398,1,44800.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,46048.0,model_complete,0.7040206789970398,1,44800.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,46464.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,47776.0,vad_stop,0.963354766368866,1,46048.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,47776.0,model_complete,0.963354766368866,1,46048.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48928.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,50784.0,vad_stop,0.9889053702354431,1,48256.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,50784.0,model_complete,0.9889053702354431,1,48256.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,62752.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,63680.0,vad_stop,0.7987582683563232,1,62080.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,63680.0,model_complete,0.7987582683563232,1,62080.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65760.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,66848.0,vad_stop,0.7040206789970398,1,65088.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,66848.0,model_complete,0.7040206789970398,1,65088.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,68512.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70176.0,vad_stop,0.3126533627510071,0,67840.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70816.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,71328.0,vad_stop,0.03396362066268921,0,67840.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,74336.0,silence_timeout,,,67840.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,80864.0,vad_start,,,
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,81088.0,vad_stop,0.8732463121414185,1,80192.0
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,81088.0,model_complete,0.8732463121414185,1,80192.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,12224.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,12640.0,vad_stop,0.9647200703620911,1,11552.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,12640.0,model_complete,0.9647200703620911,1,11552.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,22336.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,23104.0,vad_stop,0.9768373966217041,1,21664.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,23104.0,model_complete,0.9768373966217041,1,21664.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,30624.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,31424.0,vad_stop,0.9357357025146484,1,29952.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,31424.0,model_complete,0.9357357025146484,1,29952.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,42432.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,43104.0,vad_stop,0.9719371795654297,1,41760.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,43104.0,model_complete,0.9719371795654297,1,41760.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,48608.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,49440.0,vad_stop,0.9801466464996338,1,47936.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,49440.0,model_complete,0.9801466464996338,1,47936.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,51296.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,51712.0,vad_stop,0.9870367050170898,1,50624.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,51712.0,model_complete,0.9870367050170898,1,50624.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,53024.0,vad_start,,,
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,53984.0,vad_stop,0.9685282111167908,1,52352.0
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,53984.0,model_complete,0.9685282111167908,1,52352.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,9600.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,10080.0,vad_stop,0.9816229343414307,1,8928.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,10080.0,model_complete,0.9816229343414307,1,8928.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,15488.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,16000.0,vad_stop,0.9829913973808289,1,14816.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,16000.0,model_complete,0.9829913973808289,1,14816.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,22784.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,23136.0,vad_stop,0.9170267581939697,1,22112.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,23136.0,model_complete,0.9170267581939697,1,22112.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,34720.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,35008.0,vad_stop,0.01910117268562317,0,34048.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,35360.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,38304.0,vad_stop,0.9589470624923706,1,34048.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,38304.0,model_complete,0.9589470624923706,1,34048.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,38592.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,39328.0,vad_stop,0.9859892129898071,1,38304.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,39328.0,model_complete,0.9859892129898071,1,38304.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41792.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,42976.0,vad_stop,0.9604700207710266,1,41120.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,42976.0,model_complete,0.9604700207710266,1,41120.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,45024.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,47840.0,vad_stop,0.9719371795654297,1,44352.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,47840.0,model_complete,0.9719371795654297,1,44352.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,49056.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51456.0,vad_stop,0.9808988571166992,1,48384.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51456.0,model_complete,0.9808988571166992,1,48384.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,61088.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,62432.0,vad_stop,0.20124173164367676,0,60416.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,63104.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,63520.0,vad_stop,0.9573681354522705,1,60416.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,63520.0,model_complete,0.9573681354522705,1,60416.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,65056.0,vad_start,,,
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,66976.0,vad_stop,0.9884648323059082,1,64384.0
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,66976.0,model_complete,0.9884648323059082,1,64384.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,10912.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,11296.0,vad_stop,0.9808988571166992,1,10240.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,11296.0,model_complete,0.9808988571166992,1,10240.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,16832.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,17600.0,vad_stop,0.9854345321655273,1,16160.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,17600.0,model_complete,0.9854345321655273,1,16160.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,24640.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,25312.0,vad_stop,0.8934911489486694,1,23968.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,25312.0,model_complete,0.8934911489486694,1,23968.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,38848.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,40384.0,vad_stop,0.9228270053863525,1,38176.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,40384.0,model_complete,0.9228270053863525,1,38176.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41792.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,42912.0,vad_stop,0.9466100931167603,1,41120.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,42912.0,model_complete,0.9466100931167603,1,41120.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,43360.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,43968.0,vad_stop,0.9445840716362,1,42912.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,43968.0,model_complete,0.9445840716362,1,42912.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,44384.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,44864.0,vad_stop,0.8688222169876099,1,43968.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,44864.0,model_complete,0.8688222169876099,1,43968.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,45248.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,45824.0,vad_stop,0.9777120351791382,1,44864.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,45824.0,model_complete,0.9777120351791382,1,44864.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,46560.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,47968.0,vad_stop,0.9697068929672241,1,45888.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,47968.0,model_complete,0.9697068929672241,1,45888.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49568.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,50464.0,vad_stop,0.9768373966217041,1,48896.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,50464.0,model_complete,0.9768373966217041,1,48896.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,50752.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,51872.0,vad_stop,0.9749865531921387,1,50464.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,51872.0,model_complete,0.9749865531921387,1,50464.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,56544.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,57472.0,vad_stop,0.03806141018867493,0,55872.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,59104.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,60192.0,vad_stop,0.06667384505271912,0,55872.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,61248.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,63072.0,vad_stop,0.9740077257156372,1,55872.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,63072.0,model_complete,0.9740077257156372,1,55872.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,68928.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,69504.0,vad_stop,0.8050150871276855,1,68256.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,69504.0,model_complete,0.8050150871276855,1,68256.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,69920.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70368.0,vad_stop,0.9836374521255493,1,69504.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70368.0,model_complete,0.9836374521255493,1,69504.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,71392.0,vad_start,,,
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73088.0,vad_stop,0.9854345321655273,1,70720.0
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73088.0,model_complete,0.9854345321655273,1,70720.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,9408.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,9728.0,vad_stop,0.970842719078064,1,8736.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,9728.0,model_complete,0.970842719078064,1,8736.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,16032.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,16256.0,vad_stop,0.5,0,15360.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,16544.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,16960.0,vad_stop,0.9485659599304199,1,15360.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,16960.0,model_complete,0.9485659599304199,1,15360.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,27360.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,27712.0,vad_stop,0.7722399234771729,1,26688.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,27712.0,model_complete,0.7722399234771729,1,26688.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,38880.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,39488.0,vad_stop,0.9007586240768433,1,38208.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,39488.0,model_complete,0.9007586240768433,1,38208.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,42016.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,43008.0,vad_stop,0.04772377014160156,0,41344.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,45984.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,46464.0,vad_stop,0.04772377014160156,0,41344.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,46944.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,47648.0,vad_stop,0.13117775321006775,0,41344.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,47840.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,48352.0,vad_stop,0.9228270053863525,1,41344.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,48352.0,model_complete,0.9228270053863525,1,41344.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,49024.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,49440.0,vad_stop,0.8050150871276855,1,48352.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,49440.0,model_complete,0.8050150871276855,1,48352.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,51232.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,51680.0,vad_stop,0.007221877574920654,0,50560.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,52000.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,53056.0,vad_stop,0.9823201894760132,1,50560.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,53056.0,model_complete,0.9823201894760132,1,50560.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,61152.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,61920.0,vad_stop,0.07717296481132507,0,60480.0
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,63552.0,vad_start,,,
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,64320.0,vad_stop,0.009870380163192749,0,60480.0
|
||||
|
@@ -0,0 +1,45 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,input_valid,input_issue,history_issue,source_audio_channel,point_probability,point_prediction,point_correct,point_audio_source,point_audio_start_ms,point_audio_end_ms,point_real_audio_ms,model_input_ms,leading_zero_ms,vad_speaking,vad_last_start_ms,vad_last_stop_ms,strategy_prediction,strategy_correct,strategy_last_action_ms,strategy_last_action,strategy_probability,asr_user_segment_crosses_point,asr_agent_segment_crosses_point,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,21780,贵州省遵义市仁怀市。,1,low,completed_answer,xlsx_draft,1,,,0,0.9875311851501465,1,1,previous_completed_turn,18496.0,21780,3284.0,8000,4716.0,0,19168.0,19776.0,1,1,19776.0,model_complete,0.9793655872344971,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,29860,投诉建议。,1,low,completed_answer,xlsx_draft,1,,,0,0.5393059253692627,1,1,previous_completed_turn,26208.0,29860,3652.0,8000,4348.0,0,26880.0,27648.0,1,1,27648.0,model_complete,0.8971817493438721,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9865230321884155,1,0,previous_completed_turn,46336.0,48125,1789.0,8000,6211.0,0,46624.0,47168.0,1,0,47168.0,model_complete,0.978554368019104,1,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,64000,行驶3万公里。,1,low,completed_answer,xlsx_draft,1,,,0,0.9880069494247437,1,1,previous_completed_turn,57664.0,64000,6336.0,8000,1664.0,0,58336.0,59008.0,1,1,59008.0,model_complete,0.9816229343414307,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,1,,,0,0.9880069494247437,1,0,previous_completed_turn,74208.0,76672,2464.0,8000,5536.0,0,74752.0,75200.0,1,0,75200.0,model_complete,0.9445840716362,1,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,尽快，已经影响行车安全了。,1,low,completed_answer,xlsx_draft,1,,,0,0.9403132200241089,1,1,active_turn,86400.0,88482,2082.0,8000,5918.0,1,86752.0,86400.0,0,0,,,,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,22007,宣传。,1,low,completed_answer,xlsx_draft,1,,,0,0.9859892129898071,1,1,previous_completed_turn,18816.0,22007,3191.0,8000,4809.0,0,19488.0,19872.0,1,1,19872.0,model_complete,0.9333261251449585,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,31842,投诉建议。,1,low,completed_answer,xlsx_draft,1,,,0,0.9889053702354431,1,1,previous_completed_turn,28640.0,31842,3202.0,8000,4798.0,0,29312.0,29824.0,1,1,29824.0,model_complete,0.9801466464996338,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,52348,爱车型号是大众帕萨特，购车时间2023年10月份，行驶里程24000多公里。,1,low,completed_answer,xlsx_draft,1,,,0,0.9893293380737305,1,1,previous_completed_turn,48256.0,52348,4092.0,8000,3908.0,0,48928.0,50784.0,1,1,50784.0,model_complete,0.9889053702354431,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,轮胎裂纹。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9522762298583984,1,0,previous_completed_turn,62080.0,65298,3218.0,8000,4782.0,0,62752.0,63680.0,1,0,63680.0,model_complete,0.7987582683563232,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,轮胎裂纹同意更换新轮胎。但是,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.8498003482818604,1,0,previous_completed_turn,65088.0,67943,2855.0,8000,5145.0,0,65760.0,66848.0,1,0,66848.0,model_complete,0.7040206789970398,1,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,1,,,0,0.7722399234771729,1,0,active_turn,67840.0,70430,2590.0,8000,5410.0,0,68512.0,70176.0,0,1,,,,1,1,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,82941,可以。,1,low,completed_answer,xlsx_draft,1,,,0,0.9777120351791382,1,1,previous_completed_turn,80192.0,82941,2749.0,8000,5251.0,0,80864.0,81088.0,1,1,81088.0,model_complete,0.8732463121414185,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,17153,我姓宋。,1,low,completed_answer,xlsx_draft,1,,,0,0.9875311851501465,1,1,previous_completed_turn,11552.0,17153,5601.0,8000,2399.0,0,12224.0,12640.0,1,1,12640.0,model_complete,0.9647200703620911,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,24846,郑州市，河南郑州市。,1,low,completed_answer,xlsx_draft,1,,,0,0.9880069494247437,1,1,previous_completed_turn,21664.0,24846,3182.0,8000,4818.0,0,22336.0,23104.0,1,1,23104.0,model_complete,0.9768373966217041,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,33471,我的轮胎开裂了。,1,low,completed_answer,xlsx_draft,1,,,0,0.9865230321884155,1,1,previous_completed_turn,29952.0,33471,3519.0,8000,4481.0,0,30624.0,31424.0,1,1,31424.0,model_complete,0.9357357025146484,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,18027,天津市。,1,low,completed_answer,xlsx_draft,1,,,0,0.9889053702354431,1,1,previous_completed_turn,14816.0,18027,3211.0,8000,4789.0,0,15488.0,16000.0,1,1,16000.0,model_complete,0.9829913973808289,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,24768,投诉。,1,low,completed_answer,xlsx_draft,1,,,0,0.9880069494247437,1,1,previous_completed_turn,22112.0,24768,2656.0,8000,5344.0,0,22784.0,23136.0,1,1,23136.0,model_complete,0.9170267581939697,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,迈腾2023款330豪华版2.0T。出现的问题是轮购车时间是2024年5月份，里程是6851公里。,0,low,unfinished_expression,xlsx_draft,1,,,0,0.8498003482818604,1,0,active_turn,48384.0,51080,2696.0,8000,5304.0,1,49056.0,47840.0,0,1,,,,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9897372126579285,1,0,previous_completed_turn,64384.0,68313,3929.0,8000,4071.0,0,65056.0,66976.0,1,0,66976.0,model_complete,0.9884648323059082,1,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,13540,辽宁葫芦岛。,1,low,completed_answer,xlsx_draft,1,,no_prior_user_asr_segment,0,0.9816229343414307,1,1,previous_completed_turn,10240.0,13540,3300.0,8000,4700.0,0,10912.0,11296.0,1,1,11296.0,model_complete,0.9808988571166992,0,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,27520,投诉建议。,1,low,completed_answer,xlsx_draft,1,,,0,0.9875311851501465,1,1,previous_completed_turn,23968.0,27520,3552.0,8000,4448.0,0,24640.0,25312.0,1,1,25312.0,model_complete,0.8934911489486694,0,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,60790,我的轮胎行驶28000公里。,0,low,unfinished_expression,xlsx_draft,1,,,0,0.044268637895584106,0,1,active_turn,55872.0,60790,4918.0,8000,3082.0,0,59104.0,60192.0,0,1,,,,1,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70745,轮胎开裂,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9673051834106445,1,0,previous_completed_turn,69504.0,70745,1241.0,8000,6759.0,0,69920.0,70368.0,1,0,70368.0,model_complete,0.9836374521255493,1,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,18842,苏州昆山。,1,low,completed_answer,xlsx_draft,1,,,0,0.975929319858551,1,1,previous_completed_turn,15360.0,18842,3482.0,8000,4518.0,0,16544.0,16960.0,1,1,16960.0,model_complete,0.9485659599304199,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,29619,投诉。,1,low,completed_answer,xlsx_draft,1,,,0,0.9485659599304199,1,1,previous_completed_turn,26688.0,29619,2931.0,8000,5069.0,0,27360.0,27712.0,1,1,27712.0,model_complete,0.7722399234771729,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,40701,帕萨特,0,low,mid_turn_pause,xlsx_draft,1,,,0,0.9875311851501465,1,0,previous_completed_turn,38208.0,40701,2493.0,8000,5507.0,0,38880.0,39488.0,1,0,39488.0,model_complete,0.9007586240768433,1,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,54600,24年11月份购买的帕萨特，15000公里左右。,1,low,completed_answer,xlsx_draft,1,,,0,0.9854345321655273,1,1,previous_completed_turn,50560.0,54600,4040.0,8000,3960.0,0,52000.0,53056.0,1,1,53056.0,model_complete,0.9823201894760132,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,65819,四条轮胎，最近我在。,0,low,unfinished_expression,xlsx_draft,1,,,0,0.011094629764556885,0,1,active_turn,60480.0,65819,5339.0,8000,2661.0,0,63552.0,64319.99999999999,0,1,,,,1,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,1,,,0,0.9842595458030701,1,0,previous_completed_turn,41760.0,47187,5427.0,8000,2573.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,52326,我的轮胎开裂了。,0,,mid_turn_pause,,1,,,0,0.9875311851501465,1,0,previous_completed_turn,50624.0,52326,1702.0,8000,6298.0,0,51296.0,51712.0,1,0,51712.0,model_complete,0.9870367050170898,1,0,unconfirmed_manual
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48348,爱车型号是大众帕萨特，购车时间2023年10月份,0,,mid_turn_pause,,1,,,0,0.9893293380737305,1,0,previous_completed_turn,46048.0,48348,2300.0,8000,5700.0,0,46464.0,47776.0,1,0,47776.0,model_complete,0.963354766368866,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,44810,帕萨特,0,,mid_turn_pause,,1,,,0,0.9793655872344971,1,0,previous_completed_turn,41760.0,44810,3050.0,8000,4950.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,49500,帕萨特L2025款2025年3月购车,1,,completed_turn,,1,,,0,0.9808988571166992,1,1,previous_completed_turn,47936.0,49500,1564.0,8000,6436.0,0,48608.0,49440.0,1,1,49440.0,model_complete,0.9801466464996338,0,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,54318,行驶里程29000公里。,1,,completed_turn,,1,,,0,0.9848582744598389,1,1,previous_completed_turn,52352.0,54318,1966.0,8000,6034.0,0,53024.0,53984.0,1,1,53984.0,model_complete,0.9685282111167908,0,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,45581,帕萨特,0,,mid_turn_pause,,1,,,0,0.5393059253692627,1,0,previous_completed_turn,41760.0,45581,3821.0,8000,4179.0,0,42432.0,43104.0,1,0,43104.0,model_complete,0.9719371795654297,1,0,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,1,,,0,0.9884648323059082,1,0,previous_completed_turn,38304.0,41219,2915.0,8000,5085.0,0,38592.0,39328.0,1,0,39328.0,model_complete,0.9859892129898071,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,19620,,1,,completed_turn,,1,,,0,0.9865230321884155,1,1,previous_completed_turn,16160.0,19620,3460.0,8000,4540.0,0,16832.0,17600.0,1,1,17600.0,model_complete,0.9854345321655273,0,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41207,我是2023年,0,,mid_turn_pause,,1,,,0,0.8817123174667358,1,0,previous_completed_turn,38176.0,41207,3031.0,8000,4969.0,0,38848.0,40384.0,1,0,40384.0,model_complete,0.9228270053863525,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,1,,,0,0.9604700207710266,1,0,previous_completed_turn,45888.0,49204,3316.0,8000,4684.0,0,46560.0,47968.0,1,0,47968.0,model_complete,0.9697068929672241,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,58424,我的轮胎,0,,mid_turn_pause,,1,,,0,0.03147178888320923,0,1,active_turn,55872.0,58424,2552.0,8000,5448.0,0,56544.0,57472.0,0,1,,,,1,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73640,轮胎开裂，行驶28000公里，轮胎开裂,0,,mid_turn_pause,,1,,,0,0.9893293380737305,1,0,previous_completed_turn,70720.0,73640,2920.0,8000,5080.0,0,71392.0,73088.0,1,0,73088.0,model_complete,0.9854345321655273,1,0,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,11531,姓赵,1,,completed_turn,,1,,no_prior_user_asr_segment,0,0.9870367050170898,1,1,previous_completed_turn,8736.0,11531,2795.0,8000,5205.0,0,9408.0,9728.0,1,1,9728.0,model_complete,0.970842719078064,0,0,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,62828,四条轮胎,0,,mid_turn_pause,,1,,,0,0.06916707754135132,0,1,active_turn,60480.0,62828,2348.0,8000,5652.0,0,61152.0,61920.0,0,1,,,,1,0,unconfirmed_manual
|
||||
|
@@ -0,0 +1,544 @@
|
||||
{
|
||||
"total_points": 44,
|
||||
"eligible_points": 44,
|
||||
"official_verified_points": 0,
|
||||
"point_model": {
|
||||
"n": 44,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.55,
|
||||
"f1": 0.7096774193548387
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 44,
|
||||
"tn": 6,
|
||||
"fp": 16,
|
||||
"fn": 1,
|
||||
"tp": 21,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.7272727272727273,
|
||||
"false_hold_rate": 0.045454545454545456,
|
||||
"recall": 0.9545454545454546,
|
||||
"precision": 0.5675675675675675,
|
||||
"f1": 0.711864406779661
|
||||
},
|
||||
"grouped": {
|
||||
"label_status": {
|
||||
"draft": {
|
||||
"point_model": {
|
||||
"n": 29,
|
||||
"tn": 2,
|
||||
"fp": 9,
|
||||
"fn": 0,
|
||||
"tp": 18,
|
||||
"accuracy": 0.6896551724137931,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.8
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 29,
|
||||
"tn": 4,
|
||||
"fp": 7,
|
||||
"fn": 1,
|
||||
"tp": 17,
|
||||
"accuracy": 0.7241379310344828,
|
||||
"false_reply_rate": 0.6363636363636364,
|
||||
"false_hold_rate": 0.05555555555555555,
|
||||
"recall": 0.9444444444444444,
|
||||
"precision": 0.7083333333333334,
|
||||
"f1": 0.8095238095238095
|
||||
}
|
||||
},
|
||||
"unconfirmed_manual": {
|
||||
"point_model": {
|
||||
"n": 15,
|
||||
"tn": 2,
|
||||
"fp": 9,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.3076923076923077,
|
||||
"f1": 0.47058823529411764
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 15,
|
||||
"tn": 2,
|
||||
"fp": 9,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.3076923076923077,
|
||||
"f1": 0.47058823529411764
|
||||
}
|
||||
}
|
||||
},
|
||||
"call_id": {
|
||||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||||
"point_model": {
|
||||
"n": 6,
|
||||
"tn": 0,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.6666666666666666,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.8
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 6,
|
||||
"tn": 0,
|
||||
"fp": 2,
|
||||
"fn": 1,
|
||||
"tp": 3,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.25,
|
||||
"recall": 0.75,
|
||||
"precision": 0.6,
|
||||
"f1": 0.6666666666666666
|
||||
}
|
||||
},
|
||||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||||
"point_model": {
|
||||
"n": 8,
|
||||
"tn": 0,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5,
|
||||
"f1": 0.6666666666666666
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 8,
|
||||
"tn": 1,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.625,
|
||||
"false_reply_rate": 0.75,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5714285714285714,
|
||||
"f1": 0.7272727272727273
|
||||
}
|
||||
},
|
||||
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||||
"point_model": {
|
||||
"n": 9,
|
||||
"tn": 0,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 5,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5555555555555556,
|
||||
"f1": 0.7142857142857143
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 9,
|
||||
"tn": 0,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 5,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5555555555555556,
|
||||
"f1": 0.7142857142857143
|
||||
}
|
||||
},
|
||||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||||
"point_model": {
|
||||
"n": 5,
|
||||
"tn": 0,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 2,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.4,
|
||||
"f1": 0.5714285714285714
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 5,
|
||||
"tn": 1,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 2,
|
||||
"accuracy": 0.6,
|
||||
"false_reply_rate": 0.6666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5,
|
||||
"f1": 0.6666666666666666
|
||||
}
|
||||
},
|
||||
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||||
"point_model": {
|
||||
"n": 9,
|
||||
"tn": 2,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 3,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 0.6666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.42857142857142855,
|
||||
"f1": 0.6
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 9,
|
||||
"tn": 2,
|
||||
"fp": 4,
|
||||
"fn": 0,
|
||||
"tp": 3,
|
||||
"accuracy": 0.5555555555555556,
|
||||
"false_reply_rate": 0.6666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.42857142857142855,
|
||||
"f1": 0.6
|
||||
}
|
||||
},
|
||||
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||||
"point_model": {
|
||||
"n": 7,
|
||||
"tn": 2,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.8571428571428571,
|
||||
"false_reply_rate": 0.3333333333333333,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.8,
|
||||
"f1": 0.8888888888888888
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 7,
|
||||
"tn": 2,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.8571428571428571,
|
||||
"false_reply_rate": 0.3333333333333333,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.8,
|
||||
"f1": 0.8888888888888888
|
||||
}
|
||||
}
|
||||
},
|
||||
"reason": {
|
||||
"completed_answer": {
|
||||
"point_model": {
|
||||
"n": 18,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 18,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 18,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tp": 17,
|
||||
"accuracy": 0.9444444444444444,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.05555555555555555,
|
||||
"recall": 0.9444444444444444,
|
||||
"precision": 1.0,
|
||||
"f1": 0.9714285714285714
|
||||
}
|
||||
},
|
||||
"completed_turn": {
|
||||
"point_model": {
|
||||
"n": 4,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 4,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
}
|
||||
},
|
||||
"mid_turn_pause": {
|
||||
"point_model": {
|
||||
"n": 17,
|
||||
"tn": 2,
|
||||
"fp": 15,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.11764705882352941,
|
||||
"false_reply_rate": 0.8823529411764706,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 17,
|
||||
"tn": 2,
|
||||
"fp": 15,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.11764705882352941,
|
||||
"false_reply_rate": 0.8823529411764706,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
}
|
||||
},
|
||||
"unfinished_expression": {
|
||||
"point_model": {
|
||||
"n": 5,
|
||||
"tn": 2,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.4,
|
||||
"false_reply_rate": 0.6,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 5,
|
||||
"tn": 4,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.8,
|
||||
"false_reply_rate": 0.2,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"point_audio_source": {
|
||||
"active_turn": {
|
||||
"point_model": {
|
||||
"n": 7,
|
||||
"tn": 4,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 1,
|
||||
"accuracy": 0.7142857142857143,
|
||||
"false_reply_rate": 0.3333333333333333,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.3333333333333333,
|
||||
"f1": 0.5
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 7,
|
||||
"tn": 6,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tp": 0,
|
||||
"accuracy": 0.8571428571428571,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
}
|
||||
},
|
||||
"previous_completed_turn": {
|
||||
"point_model": {
|
||||
"n": 37,
|
||||
"tn": 0,
|
||||
"fp": 16,
|
||||
"fn": 0,
|
||||
"tp": 21,
|
||||
"accuracy": 0.5675675675675675,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5675675675675675,
|
||||
"f1": 0.7241379310344828
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 37,
|
||||
"tn": 0,
|
||||
"fp": 16,
|
||||
"fn": 0,
|
||||
"tp": 21,
|
||||
"accuracy": 0.5675675675675675,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5675675675675675,
|
||||
"f1": 0.7241379310344828
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"alignment_issue_points": 2,
|
||||
"history_aligned_subset": {
|
||||
"point_model": {
|
||||
"n": 42,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 20,
|
||||
"accuracy": 0.5714285714285714,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5263157894736842,
|
||||
"f1": 0.6896551724137931
|
||||
},
|
||||
"audio_strategy": {
|
||||
"n": 42,
|
||||
"tn": 6,
|
||||
"fp": 16,
|
||||
"fn": 1,
|
||||
"tp": 19,
|
||||
"accuracy": 0.5952380952380952,
|
||||
"false_reply_rate": 0.7272727272727273,
|
||||
"false_hold_rate": 0.05,
|
||||
"recall": 0.95,
|
||||
"precision": 0.5428571428571428,
|
||||
"f1": 0.6909090909090909
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"pipecat_version": "1.12.0",
|
||||
"onnxruntime_version": "1.24.4",
|
||||
"vad_params": {
|
||||
"confidence": 0.7,
|
||||
"start_secs": 0.2,
|
||||
"stop_secs": 0.2,
|
||||
"min_volume": 0.6
|
||||
},
|
||||
"smart_turn_params": {
|
||||
"stop_secs": 3,
|
||||
"pre_speech_ms": 500,
|
||||
"max_duration_secs": 8
|
||||
},
|
||||
"frame_ms": 32,
|
||||
"media_clock": true,
|
||||
"cpu_count": 1,
|
||||
"threshold": 0.5,
|
||||
"model_sha256": "2bb026316b14a660486a75b1733cd3fbab8c2fd0314dc9af7be49f8cca967e4f",
|
||||
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||||
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
||||
"audio_inputs": [
|
||||
{
|
||||
"call_id": "2c83253a-7977-4e3c-a387-0aa1ea06fbe1",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1"
|
||||
},
|
||||
{
|
||||
"call_id": "40ae686c-8196-4d7b-b7ad-3e12a427ebb6",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d"
|
||||
},
|
||||
{
|
||||
"call_id": "872a1e67-873a-4c49-a598-b38a670afcad",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5"
|
||||
},
|
||||
{
|
||||
"call_id": "a266b9ef-5e47-4434-a04c-ce4eacd52c19",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46"
|
||||
},
|
||||
{
|
||||
"call_id": "cb3238dc-3946-42a5-aad8-9b9a54226363",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60"
|
||||
},
|
||||
{
|
||||
"call_id": "e304f740-85d4-427e-9fd6-c5474e9cd737",
|
||||
"sample_rate": 8000,
|
||||
"channels": 2,
|
||||
"selected_user_channel": 0,
|
||||
"audio_sha256": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
||||
}
|
||||
],
|
||||
"replay_source_sha256": {
|
||||
"pipecat.audio.vad.silero": "9eec0ae79d63341e010fd1351031f2ea73a76204bc9ade57d3107ba5d31740a3",
|
||||
"pipecat.audio.turn.smart_turn.local_smart_turn_v3": "8bd3bc15ed96fd54539ec63400ce64ce86ce82a388d0aa467e641c9a27a2f970",
|
||||
"pipecat.audio.turn.smart_turn.base_smart_turn": "8c5c11aa3a72abb9d37447baa58475126bee7eb5932d5c8a17ee56d858be6c8a"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"point_model": "Non-mutating diagnostic inference at exact decision_ms. Active cache origin is used; after replay completed a turn, last completed origin is used and explicitly flagged. These extra queries never change replay state.",
|
||||
"audio_strategy": "Whether audio-only EOT is latched at decision_ms: actual VAD stops trigger inference; resumed VAD speech clears the latch; incomplete preserves cache; complete clears cache; silence timeout forces completion. It does not measure whether an earlier wrong EOT happened before speech resumed.",
|
||||
"event_order": "Audio appended with previous speaking flag, then queued VAD state changes handled. 32ms frames; no future or partial frames fed to VAD before a checkpoint.",
|
||||
"silero_reset": "Pipecat periodic five-second Silero state resets use media clock, not processing wall time.",
|
||||
"asr": "No realtime ASR return/finalization logs; transcript wait gate, external 5s controller timeout and ASR fallback are not simulated. History only audits point validity and speaker overlap, never enters model input.",
|
||||
"labels": "29 xlsx_draft and 15 no annotator/confidence; exploratory results only, no confirmed-label production accuracy claim.",
|
||||
"channel": "User channel 0 selected by RMS contrasts on non-overlapping ASR user/agent segments; echo/overlap can remain.",
|
||||
"invalid_points": "No detected VAD origin/prediction: excluded from metrics. Missing prior ASR user segments are alignment issues, retained in exploratory audio metrics and separately reported.",
|
||||
"scope": "Selected point classification/state snapshots; no complete turn boundaries, interval-level interruption rates or end latency claims."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
本报告为旧版结束信号回放诊断；最新结果请看 work-data-can-reply，仅评价标注点 can_reply_now。
|
||||
Smart Turn v3.2 / Pipecat 工作数据回测
|
||||
运行日期：2026-10-12
|
||||
|
||||
范围与输入
|
||||
6 通录音、44 个标注判断点，人工标签 0/1 各 22 个。原始标注未修改。
|
||||
使用 Pipecat 1.12.0 实际 Silero VAD、BaseSmartTurn 缓存及 LocalSmartTurnAnalyzerV3 推理实现。
|
||||
用户音频使用原始 8 kHz 双声道录音的第 1 声道（代码索引 0），通过转写角色区间的能量对比选择。
|
||||
VAD：confidence=0.7，min_volume=0.6，start_secs=0.2，stop_secs=0.2。
|
||||
Smart Turn：pre_speech_ms=500，max_duration_secs=8，stop_secs=3，阈值 probability > 0.5。
|
||||
32 ms 音频帧，媒体时钟；VAD 与周期性状态重置使用媒体时间。模型推理前重采样到 16 kHz，不足 8 秒前补零。
|
||||
|
||||
两个输出的含义
|
||||
strategy_prediction 是标注时刻是否保留音频结束信号：VAD 停止时自动推理，模型拒绝保留缓存，模型通过清空缓存，续说重置静音计时与结束信号，3 秒非说话累计超时强制结束。
|
||||
point_probability 是在准确 decision_ms 额外请求的诊断分数。该请求不改变自然策略回放。7 个点使用仍活跃的话轮起点；另外 37 个点自然策略已完成，诊断请求使用最近已完成片段的起点延伸至该标注时刻，并明确标为 previous_completed_turn。
|
||||
因此 point_probability 不一定是线上实际触发时的概率；分析线上音频策略应看 strategy_prediction、strategy_last_action_ms、strategy_probability 和 events.csv。
|
||||
|
||||
按当前人工标签进行探索性比较
|
||||
音频策略：27/44 正确，点级准确率 61.36%。
|
||||
人工允许接话 22 个：正确放行 21 个，误阻止 1 个，检出率 95.45%。
|
||||
人工要求等待 22 个：正确等待 6 个，误放行 16 个，误放行率 72.73%。
|
||||
额外标注点诊断查询：26/44 正确，误放行 18/22，误阻止 0/22；不能将此分数等同自然策略线上行为。
|
||||
|
||||
按标签来源分组（音频策略）
|
||||
xlsx_draft：29 点，正确 21 点，误放行 7/11，误阻止 1/18。
|
||||
未填标注员/置信度的新增标注：15 点，正确 6 点，误放行 9/11，误阻止 0/4。
|
||||
数据中没有明确的回听确认状态，以上均为探索性结果，不属于正式准确率验收。
|
||||
|
||||
错误分析
|
||||
1. 17 个 mid_turn_pause 中误放行 15 个；5 个 unfinished_expression 中误放行 1 个。
|
||||
2. 16 个误放行点最后的结束信号全部为 model_complete，没有由 silence_timeout 导致。当前主要失败发生在模型接受短暂停顿时，延长 3 秒兜底本身不会阻止这些模型提前通过。
|
||||
3. 模型一旦提前给出 Complete，缓存会清空；这时就无法继续使用原本应保留的完整用户话轮起点。后续续说从新检测的发言起点重新累计。
|
||||
4. 872a1e67 通话 44.810 秒标注“帕萨特”为等待，策略在 43.104 秒已判 Complete，实际事件概率 0.97194；45.581 秒和 47.187 秒标注点仍保留同一结束信号。三个点是相关观察，不是三个独立静音区间错误。
|
||||
5. 40ae686c 通话 67.943 秒标注等待，策略在 66.848 秒已判 Complete，实际事件概率 0.70402。
|
||||
6. 唯一误阻止：2c83253a 通话 88.482 秒，标签允许回复，但 VAD 在该点仍是 speaking，策略还未触发停止推理；额外诊断概率为 0.95403。这是触发时机与标签时刻的差异。
|
||||
|
||||
数据核对发现
|
||||
cb3238dc 的 13.540 秒标签文本为“辽宁葫芦岛”，但该文本的用户转写从 16.600 秒开始；应优先回听核对判断点或文本。
|
||||
e304f740 的 11.531 秒有“姓赵”的标签和 VAD 音频起点，但 asr_history 没有对应的早期用户记录。
|
||||
上述两个点保留音频预测，history_issue 单独标记；排除它们后的 42 点策略准确率为 59.52%，误放行仍为 16/22。
|
||||
第 1 声道可能仍有机器人回声或重叠语音，未做基于未来转写的消音或人声分离。
|
||||
|
||||
解释边界与后续
|
||||
没有 ASR 返回/最终确认日志，因此未模拟默认 wait_for_transcript=True 的转写等待、无 VAD 转写兜底及 UserTurnController 的 5 秒超时。本次是音频检测部分回放。
|
||||
strategy_prediction 是选定点的状态快照；用户恢复说话后信号会被清除，快照可能看不到更早已经发生的错误结束。缺少完整话轮/静音区间真值，不能计算正式区间误抢话率或平均结束等待时间。
|
||||
建议先回听确认新增负例及两个对齐问题，再对更高概率阈值或更长 VAD stop_secs 重新进行完整回放。不能只改 CSV 分数阈值推算策略效果，因为判定变化会改变后续缓存起点。
|
||||
|
||||
文件
|
||||
predictions.csv：44 个点的标签、诊断分数、音频截取起点/长度、自然策略状态与事件时间。
|
||||
events.csv：VAD 起止、模型 Complete、3 秒兜底事件。
|
||||
errors.csv：任一输出与当前人工标签不一致的点；不只包含策略错误。
|
||||
summary.json：指标、分组统计、版本、参数、模型/源码/输入 SHA256 和口径说明。
|
||||
|
||||
验证
|
||||
保留首段起点并在续说时重置静音计时、8 秒截断、3 秒超时清空缓存，均通过实际 Pipecat 缓存实现测试。
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Verify replay timing assumptions against Pipecat's actual buffer logic."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
from pipecat.audio.turn.base_turn_analyzer import EndOfTurnState
|
||||
from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn, SmartTurnParams
|
||||
|
||||
import evaluate_work_data as evaluation
|
||||
|
||||
|
||||
class RecordingTurn(BaseSmartTurn):
|
||||
prediction = 0
|
||||
|
||||
def _predict_endpoint(self, audio):
|
||||
self.last_audio = audio.copy()
|
||||
return {"prediction": self.prediction, "probability": 0.9 if self.prediction else 0.1}
|
||||
|
||||
|
||||
class ReplayTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.clock = evaluation.MediaClock()
|
||||
self.time_patch = patch.object(evaluation.base_module, "time", SimpleNamespace(
|
||||
monotonic=self.clock.monotonic, perf_counter=time.perf_counter))
|
||||
self.time_patch.start()
|
||||
self.turn = RecordingTurn(sample_rate=8000, params=SmartTurnParams())
|
||||
self.turn.set_sample_rate(8000)
|
||||
self.turn.update_vad_start_secs(0.2)
|
||||
self.chunk = np.full(256, 1234, dtype=np.int16).tobytes()
|
||||
|
||||
def tearDown(self):
|
||||
if self.turn._executor:
|
||||
self.turn._executor.shutdown(wait=True)
|
||||
self.time_patch.stop()
|
||||
|
||||
def feed(self, count, speaking):
|
||||
for _ in range(count):
|
||||
self.clock.seconds += 0.032
|
||||
state = self.turn.append_audio(self.chunk, speaking)
|
||||
return state
|
||||
|
||||
def test_incomplete_keeps_first_start_and_resumed_speech_resets_silence(self):
|
||||
self.feed(10, False)
|
||||
self.feed(20, True)
|
||||
first_start = self.turn._speech_start_time
|
||||
self.feed(60, False) # 1.92 s pause, below the timeout.
|
||||
state, _ = asyncio.run(self.turn.analyze_end_of_turn())
|
||||
self.assertEqual(state, EndOfTurnState.INCOMPLETE)
|
||||
preserved_count = len(self.turn._audio_buffer)
|
||||
self.feed(1, True)
|
||||
self.assertEqual(self.turn._speech_start_time, first_start)
|
||||
self.assertEqual(len(self.turn._audio_buffer), preserved_count + 1)
|
||||
self.assertEqual(self.turn._silence_ms, 0)
|
||||
|
||||
def test_model_only_gets_last_eight_seconds_and_probe_does_not_clear(self):
|
||||
self.feed(320, True) # 10.24 s accumulated speech.
|
||||
before_start = self.turn._speech_start_time
|
||||
before_count = len(self.turn._audio_buffer)
|
||||
self.turn.prediction = 1
|
||||
state, _ = self.turn._process_speech_segment(self.turn._audio_buffer)
|
||||
self.assertEqual(state, EndOfTurnState.COMPLETE)
|
||||
self.assertEqual(len(self.turn.last_audio), 8 * 8000)
|
||||
self.assertEqual(self.turn._speech_start_time, before_start)
|
||||
self.assertEqual(len(self.turn._audio_buffer), before_count)
|
||||
|
||||
def test_three_second_silence_timeout_clears_old_origin(self):
|
||||
self.feed(20, True)
|
||||
state = self.feed(94, False) # 3.008 s at 32 ms frame granularity.
|
||||
self.assertEqual(state, EndOfTurnState.COMPLETE)
|
||||
self.assertFalse(self.turn.speech_triggered)
|
||||
self.assertEqual(self.turn._speech_start_time, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Generated
+3097
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user