wx44wx/ZNJJ-api-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): named LLM provider list via env vars, runtime selection

Browse Source 
- Config.llm_profiles: LLM_PROVIDERS declares names, each group defined by
  LLM_<NAME>_BASE_URL/_API_KEY/_MODEL (required) and optional _TIMEOUT_SECONDS/
  _MAX_RETRIES/_TEMPERATURE/_EXTRA_BODY; JSON-dict form of LLM_PROVIDERS also
  supported. LLM_PROVIDER selects the active group at runtime; empty/fake =
  offline scripted model
- removed fixed openai/kimi groups (AGENT_LLM_PROVIDER, KIMI_LLM_*)
- validate(): incomplete groups and unknown LLM_PROVIDER fail fast with
  the configured-name list
- OpenAILLMService: http client trust_env from LLM_HTTP_TRUST_ENV (default
  False - local gateways must bypass the Windows system proxy)
- build_llm_service(provider): explicit name or LLM_PROVIDER default
- .env/.env.example: deepseek + kimi groups side by side, switch by
  LLM_PROVIDER=kimi (thinking off via EXTRA_BODY reasoning_effort=none)
- eval probe scripts updated to the new config interface

Verified: 172 unit tests; live calls succeed on both providers
(deepseek 28s, kimi 6.6s, both emitting valid <state> prefixes)
This commit is contained in:
Eric Wang
2026-10-22 13:59:18 +08:00
parent e72d9c9bc0
commit e5d2a78886
6 changed files with 184 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+18 -7
View File
@@ -9,11 +9,22 @@ APP_ID=replace-with-fastgpt-app-id
# Voice demo (Pipecat /ws-product). Relative to project root, or an absolute path.
VOICE_CONFIG=config/voice.json
# LangGraph chat backend + LLM provider (作业 2/4)
# LangGraph chat backend（作业 2）
# AGENT_BACKEND=langgraph
# AGENT_LLM_PROVIDER=openai
# LLM_BASE_URL=https://api.openai.com/v1
# LLM_API_KEY=replace-with-llm-api-key
# LLM_MODEL=gpt-4o-mini
# LLM_TIMEOUT_SECONDS=30
# LLM_MAX_RETRIES=2
#
# 命名 LLM 提供商组（均为 OpenAI 兼容接口）：LLM_PROVIDERS 声明名单，
# LLM_PROVIDER 选择当前生效组；缺省或 fake = 离线脚本模型。
# 每组 LLM_<名>_BASE_URL / _API_KEY / _MODEL 必填；
# 可选 _TIMEOUT_SECONDS / _MAX_RETRIES / _TEMPERATURE / _EXTRA_BODY（JSON 字符串，
# 如思考开关 KIMI 网关为 {"reasoning_effort":"none"}）。
# LLM_PROVIDERS=deepseek,kimi
# LLM_PROVIDER=deepseek
# LLM_DEEPSEEK_BASE_URL=https://api.openai.com/v1
# LLM_DEEPSEEK_API_KEY=replace-with-llm-api-key
# LLM_DEEPSEEK_MODEL=gpt-4o-mini
# LLM_KIMI_BASE_URL=http://127.0.0.1:3425/v1
# LLM_KIMI_API_KEY=magpie
# LLM_KIMI_MODEL=kimi-code-cn/kimi-for-coding
# LLM_KIMI_EXTRA_BODY={"reasoning_effort":"none"}
# 本地网关需直连（系统代理会导致 5xx）；外部网关如需代理：
# LLM_HTTP_TRUST_ENV=true
evals/prompt_compare.py
+2 -2
View File
@@ -39,13 +39,13 @@ def main():
path = args.prompt_dir / (name+'.md')
return path.read_text(encoding='utf-8') if path.exists() else original_loader(name)
registry.load_prompt = snapshot_loader
if Config.AGENT_BACKEND != 'langgraph' or Config.AGENT_LLM_PROVIDER != 'openai':
if Config.AGENT_BACKEND != 'langgraph' or not Config.LLM_PROVIDER or Config.LLM_PROVIDER == 'fake':
raise SystemExit('Requires langgraph backend and real openai provider')
args.output_dir.mkdir(parents=True, exist_ok=True)
cases = load_cases(DEFAULT_CASES_DIR, args.case)
from src.agent.prompts import load_prompt
metadata = {
'model': Config.LLM_MODEL, 'temperature': Config.LLM_TEMPERATURE,
**{k: v for k, v in (Config.llm_profile(Config.LLM_PROVIDER) or {}).items() if k in ('model', 'temperature')},
'backend': Config.AGENT_BACKEND, 'workers': args.workers,
'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
'case_hashes': {str(p.relative_to(DEFAULT_CASES_DIR)): sha256(p.read_bytes()).hexdigest()
evals/prompt_probes.py
+3 -2
View File
@@ -63,7 +63,7 @@ async def main(args):
if args.prompt_dir:
import src.agent.prompts as registry
registry.load_prompt = lambda name: (args.prompt_dir / (name+'.md')).read_text(encoding='utf-8')
llm = build_llm_service('openai')
llm = build_llm_service()
semaphore = asyncio.Semaphore(4)
async def run(case):
async with semaphore:
@@ -107,7 +107,8 @@ async def main(args):
results = await asyncio.gather(*(run(c) for c in selected))
from src.core.config import Config
from src.agent.prompts import load_prompt
output = dict(model=Config.LLM_MODEL, temperature=Config.LLM_TEMPERATURE,
profile = Config.llm_profile(Config.LLM_PROVIDER) or {}
output = dict(model=profile.get('model'), temperature=profile.get('temperature'),
execution_mode='runtime' if args.runtime else 'raw_prompt',
prompt_hashes={name:sha256(load_prompt(name).encode('utf-8')).hexdigest() for name in (COLLECTION,EXTRACTION)},
passed=sum(r['passed'] for r in results), total=len(results), results=results)
evals/time_boundary.py
+1 -1
View File
@@ -26,7 +26,7 @@ def parsed_time(value):
async def main(output):
llm = build_llm_service('openai')
llm = build_llm_service()
cases = []
for delay in (75,135):
cases.append(dict(name=f'anchored_cross_midnight_delay_{delay}',now=ORIGIN+timedelta(minutes=delay),
src/agent/llm.py
+46 -14
View File
@@ -5,7 +5,8 @@ fake 与真实 provider 实现同一 ``LLMService`` Protocol，节点不感知
- ``OpenAILLMService``：真实 provider（TODO 关键代码）——
超时、取消、有限重试、模型配置。
``build_llm_service`` 按 ``Config.AGENT_LLM_PROVIDER`` 选择实现。
``build_llm_service`` 按命名提供商列表（Config.LLM_PROVIDERS / LLM_PROVIDER）
选择实现，见 core.config。
"""
from collections.abc import Awaitable, Callable
@@ -92,6 +93,7 @@ class OpenAILLMService:
timeout_seconds: float = 30.0,
max_retries: int = 2,
temperature: float | None = None,
extra_body: dict | None = None,
):
self._model = model
self._api_key = api_key
@@ -99,17 +101,24 @@ class OpenAILLMService:
self._timeout_seconds = timeout_seconds
self._max_retries = max_retries
self._temperature = temperature
# 附加请求体（如思考开关），原样合并进 chat.completions.create 参数。
self._extra_body = dict(extra_body or {})
self._client = None
def _get_client(self):
# 延迟构造：fake provider 使用时无需安装/初始化真实客户端。
# AsyncOpenAI 的创建不绑定事件循环，可在首次调用前安全缓存。
# trust_env=False：不走系统代理，本地网关直连（代理曾导致 503/500）。
if self._client is None:
import httpx
from openai import AsyncOpenAI
from ..core.config import Config
kwargs = {"api_key": self._api_key, "timeout": self._timeout_seconds}
if self._base_url:
kwargs["base_url"] = self._base_url
kwargs["http_client"] = httpx.AsyncClient(trust_env=Config.LLM_HTTP_TRUST_ENV)
self._client = AsyncOpenAI(**kwargs)
return self._client
@@ -130,6 +139,7 @@ class OpenAILLMService:
kwargs: dict = {"model": self._model, "messages": messages}
if self._temperature is not None:
kwargs["temperature"] = self._temperature
kwargs.update(self._extra_body)
retries = 0
while True:
try:
@@ -225,19 +235,41 @@ def workflow_fake_llm() -> "FakeLLMService":
def build_llm_service(provider: str | None = None) -> LLMService:
"""按配置构造 LLM 服务；provider 缺省读 Config.AGENT_LLM_PROVIDER。"""
"""按命名提供商列表构造 LLM 服务；provider 缺省读 Config.LLM_PROVIDER。
配置见 Config.llm_profiles（LLM_PROVIDERS 名单 + LLM_<名>_* 变量）。
provider 为空或 fake = 离线脚本模型。
"""
from ..core.config import Config
provider = provider or Config.AGENT_LLM_PROVIDER
if provider == "openai":
return OpenAILLMService(
model=Config.LLM_MODEL,
api_key=Config.LLM_API_KEY,
base_url=Config.LLM_BASE_URL,
timeout_seconds=Config.LLM_TIMEOUT_SECONDS,
max_retries=Config.LLM_MAX_RETRIES,
temperature=Config.LLM_TEMPERATURE,
)
if provider == "fake":
provider = (provider if provider is not None else Config.LLM_PROVIDER)
provider = (provider or "").strip().lower()
if not provider or provider == "fake":
return FakeLLMService(_default_fake_responder)
raise ValueError(f"Unknown AGENT_LLM_PROVIDER: {provider!r}")
profiles = Config.llm_profiles()
if provider not in profiles:
raise ValueError(
f"Unknown LLM provider {provider!r}; configured: {sorted(profiles)}"
)
profile = profiles[provider]
extra_body: dict = {}
if profile["extra_body"]:
import json as _json
try:
parsed = _json.loads(profile["extra_body"])
if isinstance(parsed, dict):
extra_body = parsed
except ValueError as exc:
raise ValueError(
f"extra_body of provider {provider!r} is not valid JSON: {exc}"
) from exc
return OpenAILLMService(
model=profile["model"],
api_key=profile["api_key"],
base_url=profile["base_url"],
timeout_seconds=profile["timeout_seconds"],
max_retries=profile["max_retries"],
temperature=profile["temperature"],
extra_body=extra_body,
)
src/core/config.py
+114 -17
View File
@@ -19,18 +19,110 @@ class Config:
# "fastgpt" 保持现有行为；"langgraph" 使用内存图 fake 回复。
AGENT_BACKEND = os.getenv("AGENT_BACKEND", "fastgpt")
# LLM provider（作业 4）：fake 为脚本化假模型，离线可运行；
# openai 为真实 provider（OpenAI 兼容接口）。
AGENT_LLM_PROVIDER = os.getenv("AGENT_LLM_PROVIDER", "fake")
LLM_MODEL = os.getenv("LLM_MODEL", "fake-reply-model")
LLM_BASE_URL = os.getenv("LLM_BASE_URL", "")
LLM_API_KEY = os.getenv("LLM_API_KEY", "")
LLM_TIMEOUT_SECONDS = float(os.getenv("LLM_TIMEOUT_SECONDS", "30"))
LLM_MAX_RETRIES = int(os.getenv("LLM_MAX_RETRIES", "2"))
# 采样温度；None 表示使用 provider 缺省。对照 FastGPT 应用配置调整。
LLM_TEMPERATURE = (
float(os.getenv("LLM_TEMPERATURE")) if os.getenv("LLM_TEMPERATURE") else None
# 命名 LLM 提供商列表（均为 OpenAI 兼容接口），两种写法二选一：
# ① JSON 字典（推荐，单变量管理）：
# LLM_PROVIDERS={"deepseek": {"base_url": "https://...", "api_key": "...",
# "model": "deepseek-v4.1-flash"},
# "kimi": {"base_url": "http://127.0.0.1:3425/v1",
# "api_key": "magpie", "model": "kimi-code-cn/kimi-for-coding",
# "extra_body": {"reasoning_effort": "none"}}}
# 每组 base_url/api_key/model 必填；timeout_seconds/max_retries/
# temperature 可选；extra_body 为对象时直接合并进请求体（思考开关等）。
# ② 逗号分隔名单 + 分散变量：
# LLM_PROVIDERS=deepseek,kimi + LLM_DEEPSEEK_BASE_URL/_API_KEY/_MODEL 等。
# 运行时选择：LLM_PROVIDER=<名>；缺省或 fake = 离线脚本模型。
LLM_PROVIDERS = os.getenv("LLM_PROVIDERS", "")
LLM_PROVIDER = os.getenv("LLM_PROVIDER", "")
# LLM 客户端是否信任系统代理（Windows 注册表）。缺省 False：
# 本地网关（127.0.0.1）必须直连；外部网关若需代理设为 true。
LLM_HTTP_TRUST_ENV = os.getenv("LLM_HTTP_TRUST_ENV", "false").lower() in (
"true", "1", "t",
)
@classmethod
def _declared_provider_names(cls) -> list[str]:
"""LLM_PROVIDERS 中声明的提供商名（JSON 字典取键，名单取逗号项）。"""
raw = cls.LLM_PROVIDERS.strip()
if not raw:
return []
if raw.startswith("{"):
import json
data = json.loads(raw)
return [str(k).strip().lower() for k in data]
return [n.strip().lower() for n in raw.split(",") if n.strip()]
@classmethod
def llm_profiles(cls) -> dict[str, dict]:
"""解析命名提供商列表为统一结构；声明了但缺必填项的组不收录
（由 validate() 统一报错）。extra_body 统一为 JSON 字符串。"""
raw = cls.LLM_PROVIDERS.strip()
if not raw:
return {}
if raw.startswith("{"):
return cls._profiles_from_json(raw)
return cls._profiles_from_prefixed(raw)
@classmethod
def _profiles_from_json(cls, raw: str) -> dict[str, dict]:
import json
try:
data = json.loads(raw)
except ValueError as exc:
raise ValueError(f"LLM_PROVIDERS is not valid JSON: {exc}") from exc
if not isinstance(data, dict):
raise ValueError("LLM_PROVIDERS JSON must be an object keyed by provider name")
profiles: dict[str, dict] = {}
for key, spec in data.items():
name = str(key).strip().lower()
if not name or not isinstance(spec, dict):
continue
base_url = str(spec.get("base_url", ""))
api_key = str(spec.get("api_key", ""))
model = str(spec.get("model", ""))
if not (base_url and api_key and model):
continue
extra_body = spec.get("extra_body", "")
if isinstance(extra_body, dict):
extra_body = json.dumps(extra_body, ensure_ascii=False)
temperature = spec.get("temperature")
profiles[name] = {
"base_url": base_url,
"api_key": api_key,
"model": model,
"timeout_seconds": float(spec.get("timeout_seconds", 30)),
"max_retries": int(spec.get("max_retries", 2)),
"temperature": float(temperature) if temperature is not None else None,
"extra_body": str(extra_body),
}
return profiles
@classmethod
def _profiles_from_prefixed(cls, raw: str) -> dict[str, dict]:
profiles: dict[str, dict] = {}
for name in cls._declared_provider_names():
prefix = f"LLM_{name.upper()}_"
base_url = os.getenv(prefix + "BASE_URL", "")
api_key = os.getenv(prefix + "API_KEY", "")
model = os.getenv(prefix + "MODEL", "")
if not (base_url and api_key and model):
continue
temperature = os.getenv(prefix + "TEMPERATURE")
profiles[name] = {
"base_url": base_url,
"api_key": api_key,
"model": model,
"timeout_seconds": float(os.getenv(prefix + "TIMEOUT_SECONDS", "30")),
"max_retries": int(os.getenv(prefix + "MAX_RETRIES", "2")),
"temperature": float(temperature) if temperature else None,
"extra_body": os.getenv(prefix + "EXTRA_BODY", ""),
}
return profiles
@classmethod
def llm_profile(cls, name: str) -> dict | None:
return cls.llm_profiles().get(name.strip().lower())
@classmethod
def validate(cls):
@@ -41,16 +133,21 @@ class Config:
raise ValueError(
f"AGENT_BACKEND must be 'fastgpt' or 'langgraph', got {cls.AGENT_BACKEND!r}"
)
if cls.AGENT_LLM_PROVIDER not in ("fake", "openai"):
profiles = cls.llm_profiles() # JSON 非法时在此抛出明确错误
incomplete = [n for n in cls._declared_provider_names() if n not in profiles]
if incomplete:
raise ValueError(
"AGENT_LLM_PROVIDER must be 'fake' or 'openai', "
f"got {cls.AGENT_LLM_PROVIDER!r}"
"LLM providers missing BASE_URL/API_KEY/MODEL: "
+ ", ".join(incomplete)
)
if cls.AGENT_LLM_PROVIDER == "openai" and (
not cls.LLM_API_KEY or not cls.LLM_BASE_URL
if (
cls.LLM_PROVIDER
and cls.LLM_PROVIDER != "fake"
and cls.LLM_PROVIDER.strip().lower() not in profiles
):
raise ValueError(
"AGENT_LLM_PROVIDER=openai requires LLM_API_KEY and LLM_BASE_URL"
f"Unknown LLM_PROVIDER {cls.LLM_PROVIDER!r}; "
f"configured: {sorted(profiles)}"
)
if cls.AGENT_BACKEND != "fastgpt":
return