feat: add DashScope custom voice enrollment flow
This commit is contained in:
Xin Wang
@@ -20,6 +20,7 @@ from schemas import (
|
||||
ModelResourceUpsert,
|
||||
)
|
||||
from services.auth import require_admin
|
||||
from services.dashscope_voice import VoiceRequest, manage_voice
|
||||
from services.interface_catalog import validate_fields
|
||||
from services.masking import mask_secrets, merge_secrets
|
||||
from services.model_resource_tester import test_model_resource
|
||||
@@ -377,3 +378,16 @@ async def delete_model_resource(
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/model-resources/dashscope/voice")
|
||||
async def manage_dashscope_voice(request: VoiceRequest, session: AsyncSession = Depends(get_session)):
|
||||
"""Create/query a voice using draft settings and server-held credentials."""
|
||||
stored = {}
|
||||
if request.resource_id:
|
||||
row = await session.get(ModelResource, request.resource_id)
|
||||
if not row or row.interface_type != "dashscope-tts":
|
||||
raise HTTPException(404, "百炼 TTS 模型资源不存在")
|
||||
stored = row.secrets or {}
|
||||
key = merge_secrets({"apiKey": request.api_key}, stored).get("apiKey", "")
|
||||
return await manage_voice(request, key)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""DashScope voice enrollment, separate from the real-time synthesis path."""
|
||||
import base64
|
||||
import re
|
||||
from typing import Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from services.pipecat.dashscope_tts import SUPPORTED_MODELS
|
||||
|
||||
|
||||
class VoiceRequest(BaseModel):
|
||||
resource_id: str | None = Field(default=None, alias="resourceId")
|
||||
api_key: str = Field(default="", alias="apiKey", max_length=4096)
|
||||
api_url: str = Field(alias="apiUrl", max_length=2048)
|
||||
model: str
|
||||
action: Literal["create_voice", "query_voice"]
|
||||
prefix: str = Field(default="assistant", pattern=r"^[A-Za-z0-9]{1,10}$")
|
||||
audio: str = Field(default="", max_length=14_000_000)
|
||||
voice_id: str = Field(default="", alias="voiceId", max_length=512)
|
||||
|
||||
|
||||
def enrollment_payload(request: VoiceRequest):
|
||||
host = urlsplit(request.api_url).hostname or ""
|
||||
allowed = host in {"dashscope.aliyuncs.com", "dashscope-intl.aliyuncs.com"} or re.fullmatch(
|
||||
r"[a-zA-Z0-9-]+\.(cn-beijing|ap-southeast-1)\.maas\.aliyuncs\.com", host)
|
||||
if not allowed:
|
||||
raise HTTPException(400, "音色复刻仅支持百炼北京或新加坡官方域名")
|
||||
if request.model not in SUPPORTED_MODELS or request.model == "cosyvoice-v1":
|
||||
raise HTTPException(400, "当前模型不支持此声音复刻流程")
|
||||
data = {"action": request.action}
|
||||
if request.action == "create_voice":
|
||||
match = re.fullmatch(r"data:audio/(wav|x-wav|mpeg|mp3|mp4|m4a|x-m4a);base64,(.+)", request.audio)
|
||||
if not match:
|
||||
raise HTTPException(400, "请上传 WAV、MP3 或 M4A 音频")
|
||||
try:
|
||||
raw = base64.b64decode(match[2], validate=True)
|
||||
except ValueError:
|
||||
raise HTTPException(400, "音频编码不合法")
|
||||
if not raw or len(raw) > 10 * 1024 * 1024:
|
||||
raise HTTPException(400, "音频必须非空且不超过 10 MB")
|
||||
data.update(target_model=request.model, prefix=request.prefix, audio=request.audio)
|
||||
else:
|
||||
if not request.voice_id:
|
||||
raise HTTPException(400, "缺少音色 ID")
|
||||
data["voice_id"] = request.voice_id
|
||||
return f"https://{host}/api/v1/services/audio/tts/customization", {"model": "voice-enrollment", "input": data}
|
||||
|
||||
|
||||
async def manage_voice(request: VoiceRequest, api_key: str):
|
||||
if not api_key:
|
||||
raise HTTPException(400, "请先填写 API Key")
|
||||
url, payload = enrollment_payload(request)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
response = await client.post(url, json=payload, headers={"Authorization": f"Bearer {api_key}"})
|
||||
if response.is_error:
|
||||
raise HTTPException(502, f"百炼音色服务请求失败（HTTP {response.status_code}），请检查凭据、模型和录音要求")
|
||||
output = response.json()["output"]
|
||||
if request.action == "create_voice":
|
||||
return {"voiceId": output["voice_id"], "status": "DEPLOYING"}
|
||||
if output.get("target_model") != request.model:
|
||||
raise HTTPException(400, "此音色绑定的模型与当前选择不一致")
|
||||
return {"voiceId": request.voice_id, "status": output["status"]}
|
||||
except (httpx.HTTPError, ValueError, KeyError, TypeError):
|
||||
raise HTTPException(502, "音色服务连接异常或返回无效，请稍后查询；创建结果不确定时不要重复创建")
|
||||
@@ -0,0 +1,25 @@
|
||||
import unittest
|
||||
from fastapi import HTTPException
|
||||
from services.dashscope_voice import VoiceRequest, enrollment_payload
|
||||
|
||||
|
||||
class VoiceEnrollmentTests(unittest.TestCase):
|
||||
def request(self, **changes):
|
||||
return VoiceRequest.model_validate({"apiUrl": "wss://dashscope.aliyuncs.com/api-ws/v1/inference", "model": "cosyvoice-v3-plus", "action": "create_voice", "audio": "data:audio/wav;base64,UklGRg==", **changes})
|
||||
|
||||
def test_creation_and_region(self):
|
||||
url, payload = enrollment_payload(self.request())
|
||||
self.assertEqual(url, "https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization")
|
||||
self.assertEqual(payload["input"]["target_model"], "cosyvoice-v3-plus")
|
||||
|
||||
def test_no_credentials_to_arbitrary_host(self):
|
||||
with self.assertRaises(HTTPException):
|
||||
enrollment_payload(self.request(apiUrl="https://evil.example"))
|
||||
|
||||
def test_query_needs_id(self):
|
||||
with self.assertRaises(HTTPException):
|
||||
enrollment_payload(self.request(action="query_voice"))
|
||||
|
||||
def test_invalid_audio(self):
|
||||
with self.assertRaises(HTTPException):
|
||||
enrollment_payload(self.request(audio="data:audio/wav;base64,???"))
|
||||
@@ -0,0 +1,61 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { modelResourcesApi } from "@/lib/api";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
|
||||
export function DashscopeVoice({ resourceId, apiKey, apiUrl, model, onApply }: {
|
||||
resourceId?: string; apiKey: string; apiUrl: string; model: string; onApply: (voice: string) => void;
|
||||
}) {
|
||||
const [audio, setAudio] = useState("");
|
||||
const [voiceId, setVoiceId] = useState("");
|
||||
const [status, setStatus] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [message, setMessage] = useState("");
|
||||
|
||||
async function upload(file?: File) {
|
||||
setAudio("");
|
||||
setMessage("");
|
||||
if (!file) return;
|
||||
const extension = file.name.split(".").pop()?.toLowerCase();
|
||||
if (!extension || !["wav", "mp3", "m4a"].includes(extension) || !file.size || file.size > 10 * 1024 * 1024) {
|
||||
setMessage("请选择不超过 10 MB 的 WAV、MP3 或 M4A 文件"); return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
const encoded = await new Promise<string>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(String(reader.result).split(",")[1]);
|
||||
reader.onerror = () => reject(new Error("读取录音失败"));
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
setAudio(`data:audio/${extension === "mp3" ? "mpeg" : extension};base64,${encoded}`);
|
||||
} catch { setMessage("读取录音失败"); }
|
||||
finally { setBusy(false); }
|
||||
}
|
||||
|
||||
async function run(action: "create_voice" | "query_voice") {
|
||||
setBusy(true); setMessage("");
|
||||
try {
|
||||
const result = await modelResourcesApi.voice({ resourceId, apiKey, apiUrl, model, action, audio: action === "create_voice" ? audio : undefined, voiceId });
|
||||
setVoiceId(result.voiceId); setStatus(result.status);
|
||||
setMessage(result.status === "OK" ? "音色已就绪，可以应用到当前配置" : result.status === "DEPLOYING" ? "音色处理中，请稍后查询状态" : "音色不可用，请检查录音要求");
|
||||
} catch (error) { setMessage(error instanceof Error ? error.message : "音色请求失败"); }
|
||||
finally { setBusy(false); }
|
||||
}
|
||||
|
||||
return <div className="space-y-3 rounded-xl border border-border p-4">
|
||||
<div className="text-sm font-medium">自定义音色 · 声音复刻</div>
|
||||
<p className="text-xs text-muted-foreground">上传本人或已获授权的清晰录音，推荐 10–20 秒，最长 60 秒，采样率至少 16 kHz。音色仅用于当前模型；切换模型需重新创建。</p>
|
||||
<Input type="file" accept=".wav,.mp3,.m4a" disabled={busy} aria-label="参考录音" onChange={event => void upload(event.target.files?.[0])} />
|
||||
<Button type="button" variant="outline" disabled={busy || !audio || Boolean(voiceId)} onClick={() => void run("create_voice")}>创建音色</Button>
|
||||
<p className="text-xs text-muted-foreground">也可以粘贴已有音色 ID，查询就绪后应用。</p>
|
||||
<Input value={voiceId} disabled={busy} aria-label="自定义音色 ID" placeholder="音色 ID" onChange={event => { setVoiceId(event.target.value); setStatus(""); setMessage(""); }} />
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button type="button" variant="outline" disabled={busy || !voiceId} onClick={() => void run("query_voice")}>查询状态</Button>
|
||||
<Button type="button" disabled={busy || status !== "OK"} onClick={() => { onApply(voiceId); setMessage("已填入音色字段，请测试连接并保存模型资源"); }}>使用此音色</Button>
|
||||
</div>
|
||||
{message && <p role="status" className="text-xs text-muted-foreground">{message}</p>}
|
||||
</div>;
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
|
||||
import { DashscopeVoice } from "@/components/models/DashscopeVoice";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
@@ -807,6 +808,19 @@ export function ComponentsModelsPage() {
|
||||
</FieldSection>
|
||||
|
||||
<FieldSection title="连接与鉴权信息" scrollable fill>
|
||||
{draft.interfaceType.startsWith("dashscope-") && (
|
||||
<a href="https://help.aliyun.com/zh/model-studio/get-api-key" target="_blank" rel="noopener noreferrer" className="text-xs text-primary underline underline-offset-4">获取百炼 API Key 与鉴权说明 ↗</a>
|
||||
)}
|
||||
{draft.interfaceType === "dashscope-tts" && (
|
||||
<DashscopeVoice
|
||||
key={`${editingId ?? "new"}:${draft.values.model}:${draft.values.apiUrl}`}
|
||||
resourceId={editingId ?? undefined}
|
||||
apiKey={String(draft.secrets.apiKey || storedSecretMasks.apiKey || "")}
|
||||
apiUrl={String(draft.values.apiUrl || "")}
|
||||
model={String(draft.values.model || "")}
|
||||
onApply={voice => { setDraft(previous => ({ ...previous, values: { ...previous.values, voice } })); setTestResult(null); }}
|
||||
/>
|
||||
)}
|
||||
{connectionFields.map((field) => (
|
||||
<DynamicField
|
||||
key={`${field.group}-${field.key}`}
|
||||
|
||||
@@ -229,6 +229,8 @@ export const interfaceDefinitionsApi = {
|
||||
};
|
||||
|
||||
export const modelResourcesApi = {
|
||||
voice: (body: { resourceId?: string; apiKey: string; apiUrl: string; model: string; action: "create_voice" | "query_voice"; audio?: string; voiceId?: string }) =>
|
||||
request<{ voiceId: string; status: string }>("/api/model-resources/dashscope/voice", { method: "POST", body: JSON.stringify(body) }),
|
||||
list: () => request<ModelResource[]>("/api/model-resources"),
|
||||
create: (body: ModelResourceUpsert) =>
|
||||
request<ModelResource>("/api/model-resources", {
|
||||
|
||||
Reference in New Issue
Block a user