mirror of
https://github.com/ZhuLinsen/daily_stock_analysis
synced 2026-09-20 10:53:33 +08:00
feat: add explicit Responses API channel routing (#2157)
This commit is contained in:
315
src/config.py
315
src/config.py
@@ -14,6 +14,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
from urllib.parse import unquote, urlparse
|
||||
@@ -63,6 +64,7 @@ from src.llm.hermes import (
|
||||
HERMES_DEFAULT_MODEL,
|
||||
HERMES_DEFAULT_PROTOCOL,
|
||||
HermesConfigIssue,
|
||||
hermes_blocked_route_candidates,
|
||||
hermes_model_info,
|
||||
is_reserved_hermes_name,
|
||||
parse_hermes_channel,
|
||||
@@ -97,6 +99,26 @@ class ConfigIssue:
|
||||
|
||||
_MANAGED_LITELLM_KEY_PROVIDERS = {"gemini", "vertex_ai", "anthropic", "openai", "deepseek"}
|
||||
SUPPORTED_LLM_CHANNEL_PROTOCOLS = ("openai", "anthropic", "gemini", "vertex_ai", "deepseek", "ollama")
|
||||
SUPPORTED_LLM_CHANNEL_API_SURFACES = ("chat_completions", "responses")
|
||||
_FALLBACK_LITELLM_MODEL_PROVIDERS = _MANAGED_LITELLM_KEY_PROVIDERS | set(SUPPORTED_LLM_CHANNEL_PROTOCOLS) | {
|
||||
"minimax",
|
||||
"cohere",
|
||||
"huggingface",
|
||||
"bedrock",
|
||||
"sagemaker",
|
||||
"azure",
|
||||
"replicate",
|
||||
"together_ai",
|
||||
"palm",
|
||||
"text-completion-openai",
|
||||
"command-r",
|
||||
"groq",
|
||||
"cerebras",
|
||||
"fireworks_ai",
|
||||
"friendliai",
|
||||
"openrouter",
|
||||
"xai",
|
||||
}
|
||||
_FALSEY_ENV_VALUES = {"0", "false", "no", "off"}
|
||||
PROMPT_CACHE_DIAGNOSTICS_LEVELS = {"off", "basic", "debug"}
|
||||
SUPPORTED_AGENT_BACKENDS = {"auto", "litellm", "codex_app_server"}
|
||||
@@ -382,6 +404,100 @@ def canonicalize_llm_channel_protocol(value: Optional[str]) -> str:
|
||||
return aliases.get(candidate, candidate)
|
||||
|
||||
|
||||
def canonicalize_llm_channel_api_surface(value: Optional[str]) -> str:
|
||||
"""Normalize an LLM channel endpoint surface label."""
|
||||
candidate = (value or "").strip().lower().replace("-", "_")
|
||||
aliases = {
|
||||
"chat": "chat_completions",
|
||||
"chat_completion": "chat_completions",
|
||||
"completions": "chat_completions",
|
||||
"response": "responses",
|
||||
"responses_api": "responses",
|
||||
}
|
||||
return aliases.get(candidate, candidate)
|
||||
|
||||
|
||||
def normalize_llm_channel_api_surface(value: Optional[str]) -> str:
|
||||
"""Return a supported endpoint surface, defaulting to Chat Completions."""
|
||||
normalized = canonicalize_llm_channel_api_surface(value)
|
||||
if normalized in SUPPORTED_LLM_CHANNEL_API_SURFACES:
|
||||
return normalized
|
||||
return "chat_completions"
|
||||
|
||||
|
||||
def is_supported_llm_channel_api_surface_value(value: Optional[str]) -> bool:
|
||||
"""Return whether a raw API surface is blank or recognized."""
|
||||
canonical = canonicalize_llm_channel_api_surface(value)
|
||||
return not canonical or canonical in SUPPORTED_LLM_CHANNEL_API_SURFACES
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_litellm_model_providers() -> frozenset[str]:
|
||||
"""Return provider identifiers from the installed LiteLLM routing enum.
|
||||
|
||||
LiteLLM adds direct providers independently of this repository. Loading
|
||||
its enum keeps channel validation aligned with the actual router instead
|
||||
of relying on a permanently incomplete local allow-list. The fallback is
|
||||
only for lightweight test stubs or a broken optional import; a production
|
||||
installation gets the complete provider set from its pinned LiteLLM.
|
||||
"""
|
||||
providers = set(_FALLBACK_LITELLM_MODEL_PROVIDERS)
|
||||
try:
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
providers.update(
|
||||
str(provider.value).strip().lower()
|
||||
for provider in LlmProviders
|
||||
if str(getattr(provider, "value", "")).strip()
|
||||
)
|
||||
except (ImportError, AttributeError, TypeError):
|
||||
logger.debug("LiteLLM provider metadata unavailable; using the compatibility fallback")
|
||||
return frozenset(providers)
|
||||
|
||||
|
||||
def get_explicit_llm_channel_model_provider(model: str) -> str:
|
||||
"""Return the explicit LiteLLM provider prefix, if the model has one.
|
||||
|
||||
A slash alone does not establish a provider: OpenAI-compatible gateways
|
||||
commonly expose provider-owned IDs such as ``Qwen/Qwen3`` or
|
||||
``deepseek-ai/DeepSeek-V3``. Only prefixes understood as LiteLLM providers
|
||||
are treated as routing declarations.
|
||||
"""
|
||||
normalized_model = (model or "").strip()
|
||||
if "/" not in normalized_model:
|
||||
return ""
|
||||
raw_prefix = normalized_model.split("/", 1)[0].lower()
|
||||
canonical_prefix = canonicalize_llm_channel_protocol(raw_prefix)
|
||||
providers = get_litellm_model_providers()
|
||||
if raw_prefix in providers:
|
||||
return raw_prefix
|
||||
if canonical_prefix in providers:
|
||||
return canonical_prefix
|
||||
return ""
|
||||
|
||||
|
||||
def apply_litellm_api_surface(model: str, api_surface: Optional[str]) -> str:
|
||||
"""Encode an explicit API surface in a LiteLLM wire model.
|
||||
|
||||
LiteLLM's ``provider/responses/model`` convention keeps the public Router
|
||||
alias stable while letting ``completion()`` bridge messages, streaming,
|
||||
tools, responses, and usage through the provider's Responses endpoint.
|
||||
"""
|
||||
normalized_model = (model or "").strip()
|
||||
if not normalized_model or normalize_llm_channel_api_surface(api_surface) != "responses":
|
||||
return normalized_model
|
||||
provider = get_explicit_llm_channel_model_provider(normalized_model)
|
||||
if provider != "openai":
|
||||
raise ValueError(
|
||||
"Responses API surface requires a normalized openai/<model> route; "
|
||||
f"got {normalized_model!r}"
|
||||
)
|
||||
provider, remainder = normalized_model.split("/", 1)
|
||||
if remainder.startswith("responses/"):
|
||||
return normalized_model
|
||||
return f"{provider}/responses/{remainder}"
|
||||
|
||||
|
||||
def resolve_llm_channel_protocol(
|
||||
protocol: Optional[str],
|
||||
*,
|
||||
@@ -443,15 +559,10 @@ def normalize_llm_channel_model(model: str, protocol: Optional[str], base_url: O
|
||||
raw_prefix, remainder = normalized_model.split("/", 1)
|
||||
prefix = raw_prefix.lower()
|
||||
canonical_prefix = canonicalize_llm_channel_protocol(prefix)
|
||||
known_providers = _MANAGED_LITELLM_KEY_PROVIDERS | set(SUPPORTED_LLM_CHANNEL_PROTOCOLS) | {
|
||||
"minimax",
|
||||
"cohere", "huggingface", "bedrock", "sagemaker", "azure",
|
||||
"replicate", "together_ai", "palm", "text-completion-openai",
|
||||
"command-r", "groq", "cerebras", "fireworks_ai", "friendliai",
|
||||
}
|
||||
if prefix in known_providers:
|
||||
providers = get_litellm_model_providers()
|
||||
if prefix in providers:
|
||||
return normalized_model
|
||||
if canonical_prefix in known_providers:
|
||||
if canonical_prefix in providers:
|
||||
return f"{canonical_prefix}/{remainder}"
|
||||
# Not a real provider prefix — add one so LiteLLM routes correctly.
|
||||
if resolved_protocol:
|
||||
@@ -463,6 +574,58 @@ def normalize_llm_channel_model(model: str, protocol: Optional[str], base_url: O
|
||||
return f"{resolved_protocol}/{normalized_model}"
|
||||
|
||||
|
||||
def find_incompatible_llm_channel_models(
|
||||
models: List[str],
|
||||
protocol: Optional[str],
|
||||
api_surface: Optional[str],
|
||||
base_url: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
"""Return models whose actual LiteLLM route conflicts with the surface.
|
||||
|
||||
Responses routing is implemented through LiteLLM's OpenAI bridge, so both
|
||||
the channel protocol and every normalized model route must resolve to the
|
||||
OpenAI provider. This is the shared invariant used by validation, runtime
|
||||
loading, diagnostics, and screening.
|
||||
"""
|
||||
if normalize_llm_channel_api_surface(api_surface) != "responses":
|
||||
return []
|
||||
resolved_protocol = resolve_llm_channel_protocol(
|
||||
protocol,
|
||||
base_url=base_url,
|
||||
models=models,
|
||||
)
|
||||
if resolved_protocol != "openai":
|
||||
return [model for model in models if (model or "").strip()]
|
||||
incompatible: List[str] = []
|
||||
for model in models:
|
||||
normalized_model = normalize_llm_channel_model(model, resolved_protocol, base_url)
|
||||
if normalized_model and get_explicit_llm_channel_model_provider(normalized_model) != "openai":
|
||||
incompatible.append(model)
|
||||
return incompatible
|
||||
|
||||
|
||||
def find_llm_channel_surface_conflicts(
|
||||
channels: List[Dict[str, Any]],
|
||||
) -> Dict[str, Tuple[str, ...]]:
|
||||
"""Return public route aliases declared with more than one API surface."""
|
||||
route_surfaces: Dict[str, set[str]] = {}
|
||||
for channel in channels:
|
||||
if not isinstance(channel, dict) or not channel.get("enabled", True):
|
||||
continue
|
||||
protocol = str(channel.get("protocol") or "")
|
||||
base_url = str(channel.get("base_url") or "")
|
||||
surface = normalize_llm_channel_api_surface(channel.get("api_surface"))
|
||||
for raw_model in channel.get("models") or []:
|
||||
model = normalize_llm_channel_model(str(raw_model), protocol, base_url)
|
||||
if model:
|
||||
route_surfaces.setdefault(model, set()).add(surface)
|
||||
return {
|
||||
model: tuple(sorted(surfaces))
|
||||
for model, surfaces in route_surfaces.items()
|
||||
if len(surfaces) > 1
|
||||
}
|
||||
|
||||
|
||||
def get_configured_llm_models(model_list: List[Dict[str, Any]]) -> List[str]:
|
||||
"""Return non-legacy model names declared in Router model_list order.
|
||||
|
||||
@@ -1315,19 +1478,15 @@ class Config:
|
||||
os.getenv('ANSPIRE_LLM_BASE_URL') or ANSPIRE_LLM_BASE_URL_DEFAULT
|
||||
).strip()
|
||||
_anspire_llm_model_env = os.getenv('ANSPIRE_LLM_MODEL', '').strip()
|
||||
anspire_channel_disabled = False
|
||||
anspire_channel_declared = False
|
||||
for _raw_channel in os.getenv('LLM_CHANNELS', '').split(','):
|
||||
if _raw_channel.strip().lower() != "anspire":
|
||||
continue
|
||||
_channel_enabled_raw = os.getenv('LLM_ANSPIRE_ENABLED')
|
||||
if _channel_enabled_raw is not None and _channel_enabled_raw.strip():
|
||||
anspire_channel_disabled = not parse_env_bool(_channel_enabled_raw, default=True)
|
||||
else:
|
||||
anspire_channel_disabled = not anspire_llm_enabled
|
||||
anspire_channel_declared = True
|
||||
break
|
||||
using_anspire_llm_legacy = bool(
|
||||
anspire_llm_enabled
|
||||
and not anspire_channel_disabled
|
||||
and not anspire_channel_declared
|
||||
and anspire_api_keys
|
||||
and not openai_api_keys
|
||||
)
|
||||
@@ -2163,6 +2322,7 @@ class Config:
|
||||
Format:
|
||||
LLM_CHANNELS=aihubmix,deepseek,gemini
|
||||
LLM_AIHUBMIX_PROTOCOL=openai
|
||||
LLM_AIHUBMIX_API_SURFACE=chat_completions
|
||||
LLM_AIHUBMIX_BASE_URL=https://aihubmix.com/v1
|
||||
LLM_AIHUBMIX_API_KEY=sk-xxx (or LLM_AIHUBMIX_API_KEYS=k1,k2)
|
||||
LLM_AIHUBMIX_MODELS=gpt-5.5,claude-sonnet-4-6
|
||||
@@ -2175,6 +2335,15 @@ class Config:
|
||||
issues: List[HermesConfigIssue] = []
|
||||
blocks_legacy_fallback = False
|
||||
blocked_hermes_routes: List[str] = []
|
||||
|
||||
def record_blocked_hermes_routes(raw_models: List[str]) -> None:
|
||||
nonlocal blocks_legacy_fallback
|
||||
blocks_legacy_fallback = True
|
||||
for raw_model in raw_models or [HERMES_DEFAULT_MODEL]:
|
||||
for route_name in hermes_blocked_route_candidates(raw_model):
|
||||
if route_name not in blocked_hermes_routes:
|
||||
blocked_hermes_routes.append(route_name)
|
||||
|
||||
for raw_name in channels_str.split(','):
|
||||
ch_name = raw_name.strip()
|
||||
if not ch_name:
|
||||
@@ -2190,6 +2359,7 @@ class Config:
|
||||
protocol_raw = os.getenv(f'LLM_{ch_upper}_PROTOCOL', '').strip()
|
||||
if ch_lower == "anspire" and not protocol_raw:
|
||||
protocol_raw = "openai"
|
||||
api_surface_raw = os.getenv(f'LLM_{ch_upper}_API_SURFACE', '').strip()
|
||||
enabled_raw = os.getenv(f'LLM_{ch_upper}_ENABLED')
|
||||
if ch_lower == "anspire" and (enabled_raw is None or not enabled_raw.strip()):
|
||||
enabled_raw = os.getenv('ANSPIRE_LLM_ENABLED')
|
||||
@@ -2216,7 +2386,45 @@ class Config:
|
||||
if anspire_model:
|
||||
raw_models = [anspire_model]
|
||||
|
||||
# Disabled channels are inert. In particular, stale values such as
|
||||
# LLM_HERMES_API_SURFACE=responses must not block valid legacy
|
||||
# deployments after Hermes has been explicitly disabled.
|
||||
if not enabled:
|
||||
_logger.info("LLM channel '%s': disabled, skipped", ch_name)
|
||||
continue
|
||||
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface_raw):
|
||||
issues.append(HermesConfigIssue(
|
||||
f"LLM_{ch_upper}_API_SURFACE",
|
||||
"invalid_api_surface",
|
||||
(
|
||||
f"Unsupported LLM API surface '{api_surface_raw}'. "
|
||||
f"Supported: {', '.join(SUPPORTED_LLM_CHANNEL_API_SURFACES)}"
|
||||
),
|
||||
))
|
||||
if is_reserved_hermes_name(ch_name):
|
||||
record_blocked_hermes_routes(raw_models)
|
||||
_logger.warning(
|
||||
"LLM_%s_API_SURFACE=%s is unsupported; channel skipped",
|
||||
ch_upper,
|
||||
api_surface_raw,
|
||||
)
|
||||
continue
|
||||
api_surface = normalize_llm_channel_api_surface(api_surface_raw)
|
||||
|
||||
if is_reserved_hermes_name(ch_name):
|
||||
if api_surface == "responses":
|
||||
issues.append(HermesConfigIssue(
|
||||
f"LLM_{ch_upper}_API_SURFACE",
|
||||
"hermes_responses_unsupported",
|
||||
"The reserved Hermes channel does not support the Responses API surface",
|
||||
))
|
||||
record_blocked_hermes_routes(raw_models)
|
||||
_logger.warning(
|
||||
"LLM_%s_API_SURFACE=responses is unsupported for reserved Hermes channel; channel skipped",
|
||||
ch_upper,
|
||||
)
|
||||
continue
|
||||
if not raw_models:
|
||||
raw_models = [HERMES_DEFAULT_MODEL]
|
||||
result = parse_hermes_channel(
|
||||
@@ -2244,6 +2452,38 @@ class Config:
|
||||
continue
|
||||
|
||||
protocol = resolve_llm_channel_protocol(protocol_raw, base_url=base_url, models=raw_models, channel_name=ch_name)
|
||||
if api_surface == "responses" and protocol != "openai":
|
||||
issues.append(HermesConfigIssue(
|
||||
f"LLM_{ch_upper}_API_SURFACE",
|
||||
"responses_requires_openai_protocol",
|
||||
"Responses API surface currently requires the openai protocol",
|
||||
))
|
||||
_logger.warning(
|
||||
"LLM_%s_API_SURFACE=responses requires protocol=openai; channel skipped",
|
||||
ch_upper,
|
||||
)
|
||||
continue
|
||||
incompatible_models = find_incompatible_llm_channel_models(
|
||||
raw_models,
|
||||
protocol,
|
||||
api_surface,
|
||||
base_url,
|
||||
)
|
||||
if incompatible_models:
|
||||
issues.append(HermesConfigIssue(
|
||||
f"LLM_{ch_upper}_MODELS",
|
||||
"responses_requires_openai_model_provider",
|
||||
(
|
||||
"Responses API surface requires every model to use the OpenAI "
|
||||
f"provider route; incompatible: {', '.join(incompatible_models[:3])}"
|
||||
),
|
||||
))
|
||||
_logger.warning(
|
||||
"LLM_%s_API_SURFACE=responses has non-OpenAI model routes (%s); channel skipped",
|
||||
ch_upper,
|
||||
", ".join(incompatible_models[:3]),
|
||||
)
|
||||
continue
|
||||
models = [normalize_llm_channel_model(m, protocol, base_url) for m in raw_models]
|
||||
|
||||
# Extra headers (JSON string, optional)
|
||||
@@ -2255,10 +2495,6 @@ class Config:
|
||||
except json.JSONDecodeError:
|
||||
_logger.warning(f"LLM_{ch_upper}_EXTRA_HEADERS: invalid JSON, ignored")
|
||||
|
||||
if not enabled:
|
||||
_logger.info(f"LLM channel '{ch_name}': disabled, skipped")
|
||||
continue
|
||||
|
||||
if protocol_raw and canonicalize_llm_channel_protocol(protocol_raw) not in SUPPORTED_LLM_CHANNEL_PROTOCOLS:
|
||||
_logger.warning(
|
||||
"LLM_%s_PROTOCOL=%s is unsupported; auto-detected protocol=%s",
|
||||
@@ -2280,6 +2516,7 @@ class Config:
|
||||
channels.append({
|
||||
'name': ch_name.lower(),
|
||||
'protocol': protocol,
|
||||
'api_surface': api_surface,
|
||||
'enabled': enabled,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys,
|
||||
@@ -2288,6 +2525,36 @@ class Config:
|
||||
})
|
||||
_logger.info(f"LLM channel '{ch_name}': {len(models)} model(s), {len(api_keys)} key(s)")
|
||||
|
||||
surface_conflicts = find_llm_channel_surface_conflicts(channels)
|
||||
if surface_conflicts:
|
||||
conflicting_models = set(surface_conflicts)
|
||||
for model, surfaces in surface_conflicts.items():
|
||||
issues.append(HermesConfigIssue(
|
||||
"LLM_CHANNELS",
|
||||
"mixed_api_surfaces_for_route",
|
||||
(
|
||||
f"LLM route alias '{model}' is declared with multiple API surfaces: "
|
||||
f"{', '.join(surfaces)}"
|
||||
),
|
||||
))
|
||||
_logger.warning(
|
||||
"LLM route alias '%s' mixes API surfaces (%s); conflicting channels skipped",
|
||||
model,
|
||||
", ".join(surfaces),
|
||||
)
|
||||
channels = [
|
||||
channel
|
||||
for channel in channels
|
||||
if not {
|
||||
normalize_llm_channel_model(
|
||||
str(model),
|
||||
str(channel.get("protocol") or ""),
|
||||
str(channel.get("base_url") or ""),
|
||||
)
|
||||
for model in channel.get("models") or []
|
||||
}.intersection(conflicting_models)
|
||||
]
|
||||
|
||||
return channels, issues, blocks_legacy_fallback, blocked_hermes_routes
|
||||
|
||||
@classmethod
|
||||
@@ -2298,6 +2565,12 @@ class Config:
|
||||
- LiteLLM providers: https://docs.litellm.ai/docs/providers
|
||||
- LiteLLM model_list 语义: https://docs.litellm.ai/docs/proxy/configs#the-model_list-key
|
||||
"""
|
||||
surface_conflicts = find_llm_channel_surface_conflicts(channels)
|
||||
if surface_conflicts:
|
||||
raise ValueError(
|
||||
"LLM route aliases cannot mix API surfaces: "
|
||||
+ ", ".join(sorted(surface_conflicts))
|
||||
)
|
||||
model_list: List[Dict[str, Any]] = []
|
||||
for ch in channels:
|
||||
hermes_refs = {
|
||||
@@ -2309,6 +2582,8 @@ class Config:
|
||||
for api_key in ch['api_keys']:
|
||||
model_ref = hermes_refs.get(str(model_name))
|
||||
wire_model = str((model_ref or {}).get("wire_model") or model_name)
|
||||
api_surface = normalize_llm_channel_api_surface(ch.get("api_surface"))
|
||||
wire_model = apply_litellm_api_surface(wire_model, api_surface)
|
||||
litellm_params: Dict[str, Any] = {
|
||||
'model': wire_model,
|
||||
}
|
||||
@@ -2331,6 +2606,8 @@ class Config:
|
||||
entry["model_info"] = hermes_model_info(
|
||||
str((model_ref or {}).get("display_model") or "")
|
||||
)
|
||||
elif api_surface == "responses":
|
||||
entry["model_info"] = {"dsa_api_surface": "responses"}
|
||||
model_list.append(entry)
|
||||
return model_list
|
||||
|
||||
|
||||
@@ -393,11 +393,12 @@ def build_provider_cache_route_context(
|
||||
_model_list_api_base(model, model_list),
|
||||
)
|
||||
family = infer_provider_family(model=model, provider=provider, api_base=api_base)
|
||||
configured_api_surface = _model_list_api_surface(model, model_list)
|
||||
return ProviderCacheRouteContext(
|
||||
model=model,
|
||||
provider=provider or family,
|
||||
api_base=api_base,
|
||||
api_surface=_infer_api_surface(family, api_base),
|
||||
api_surface=configured_api_surface or _infer_api_surface(family, api_base),
|
||||
gateway=_infer_gateway(api_base, family),
|
||||
cloud_platform=_infer_cloud_platform(api_base, family),
|
||||
call_type=call_type,
|
||||
@@ -727,6 +728,38 @@ def _model_list_api_base(model: str, model_list: Optional[List[Dict[str, Any]]])
|
||||
return None
|
||||
|
||||
|
||||
def _model_list_api_surface(model: str, model_list: Optional[List[Dict[str, Any]]]) -> Optional[ApiSurface]:
|
||||
"""Return the endpoint surface attached to a matching Router deployment."""
|
||||
normalized_model = (model or "").strip()
|
||||
if not normalized_model or not model_list:
|
||||
return None
|
||||
surfaces: set[str] = set()
|
||||
for entry in model_list:
|
||||
if not isinstance(entry, Mapping):
|
||||
continue
|
||||
params = entry.get("litellm_params", {}) or {}
|
||||
if not isinstance(params, Mapping):
|
||||
params = {}
|
||||
names = {
|
||||
str(entry.get("model_name") or "").strip(),
|
||||
str(params.get("model") or "").strip(),
|
||||
}
|
||||
if normalized_model not in names:
|
||||
continue
|
||||
model_info = entry.get("model_info", {}) or {}
|
||||
if not isinstance(model_info, Mapping):
|
||||
model_info = {}
|
||||
surface = str(model_info.get("dsa_api_surface") or "chat_completions").strip().lower()
|
||||
surfaces.add(surface)
|
||||
if len(surfaces) == 1:
|
||||
surface = next(iter(surfaces))
|
||||
if surface in {"responses", "chat_completions"}:
|
||||
return surface
|
||||
if len(surfaces) > 1:
|
||||
return "unknown"
|
||||
return None
|
||||
|
||||
|
||||
def _first_non_empty(*values: Any) -> Optional[str]:
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
|
||||
@@ -17,6 +17,10 @@ from src.config import (
|
||||
_uses_direct_env_provider,
|
||||
channel_allows_empty_api_key,
|
||||
get_configured_llm_models,
|
||||
is_supported_llm_channel_api_surface_value,
|
||||
find_incompatible_llm_channel_models,
|
||||
find_llm_channel_surface_conflicts,
|
||||
normalize_llm_channel_api_surface,
|
||||
normalize_llm_channel_model,
|
||||
parse_env_bool,
|
||||
resolve_llm_channel_protocol,
|
||||
@@ -844,6 +848,10 @@ class GenerationBackendStatusService:
|
||||
protocol_raw = (effective_map.get(f"{prefix}_PROTOCOL") or "").strip()
|
||||
if lower == "anspire" and not protocol_raw:
|
||||
protocol_raw = "openai"
|
||||
api_surface_raw = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface_raw):
|
||||
continue
|
||||
api_surface = normalize_llm_channel_api_surface(api_surface_raw)
|
||||
|
||||
api_keys = cls._split_csv(effective_map.get(f"{prefix}_API_KEYS") or "")
|
||||
single_key = (effective_map.get(f"{prefix}_API_KEY") or "").strip()
|
||||
@@ -857,6 +865,8 @@ class GenerationBackendStatusService:
|
||||
raw_models = [(effective_map.get("ANSPIRE_LLM_MODEL") or ANSPIRE_LLM_MODEL_DEFAULT).strip()]
|
||||
|
||||
if is_reserved_hermes_name(name):
|
||||
if api_surface == "responses":
|
||||
continue
|
||||
result = parse_hermes_channel(
|
||||
enabled=True,
|
||||
protocol=protocol_raw or HERMES_DEFAULT_PROTOCOL,
|
||||
@@ -871,6 +881,10 @@ class GenerationBackendStatusService:
|
||||
continue
|
||||
|
||||
protocol = resolve_llm_channel_protocol(protocol_raw, base_url=base_url, models=raw_models, channel_name=name)
|
||||
if api_surface == "responses" and protocol != "openai":
|
||||
continue
|
||||
if find_incompatible_llm_channel_models(raw_models, protocol, api_surface, base_url):
|
||||
continue
|
||||
models = [normalize_llm_channel_model(model, protocol, base_url) for model in raw_models]
|
||||
if not api_keys and channel_allows_empty_api_key(protocol, base_url):
|
||||
api_keys = [""]
|
||||
@@ -882,6 +896,7 @@ class GenerationBackendStatusService:
|
||||
{
|
||||
"name": lower,
|
||||
"protocol": protocol,
|
||||
"api_surface": api_surface,
|
||||
"enabled": True,
|
||||
"base_url": base_url,
|
||||
"api_keys": api_keys,
|
||||
@@ -889,7 +904,14 @@ class GenerationBackendStatusService:
|
||||
"extra_headers": extra_headers,
|
||||
}
|
||||
)
|
||||
return channels
|
||||
surface_conflicts = set(find_llm_channel_surface_conflicts(channels))
|
||||
if not surface_conflicts:
|
||||
return channels
|
||||
return [
|
||||
channel
|
||||
for channel in channels
|
||||
if not set(channel.get("models") or []).intersection(surface_conflicts)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _parse_json_object(value: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
@@ -17,9 +17,9 @@ import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from src.config import Config, get_config
|
||||
from src.config import Config, channel_allows_empty_api_key, get_config
|
||||
from src.llm.hermes import route_has_hermes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -227,8 +227,29 @@ def _resolve_vision_model() -> str:
|
||||
return model
|
||||
|
||||
|
||||
def _matching_vision_deployments(model: str, cfg: Config) -> List[Dict[str, Any]]:
|
||||
"""Return configured LiteLLM deployments for a public vision route."""
|
||||
normalized_model = (model or "").strip()
|
||||
if not normalized_model:
|
||||
return []
|
||||
return [
|
||||
entry
|
||||
for entry in (getattr(cfg, "llm_model_list", []) or [])
|
||||
if isinstance(entry, dict)
|
||||
and str(entry.get("model_name") or "").strip() == normalized_model
|
||||
and isinstance(entry.get("litellm_params"), dict)
|
||||
]
|
||||
|
||||
|
||||
def _get_api_keys_for_model(model: str, cfg: Config) -> List[str]:
|
||||
"""Return available API keys for the given litellm model."""
|
||||
deployment_keys: List[str] = []
|
||||
for deployment in _matching_vision_deployments(model, cfg):
|
||||
key = str((deployment.get("litellm_params") or {}).get("api_key") or "").strip()
|
||||
if key and len(key) >= 8 and key not in deployment_keys:
|
||||
deployment_keys.append(key)
|
||||
if deployment_keys:
|
||||
return deployment_keys
|
||||
if model.startswith("gemini/") or model.startswith("vertex_ai/"):
|
||||
return [k for k in cfg.gemini_api_keys if k and len(k) >= 8]
|
||||
if model.startswith("anthropic/"):
|
||||
@@ -236,6 +257,16 @@ def _get_api_keys_for_model(model: str, cfg: Config) -> List[str]:
|
||||
return [k for k in cfg.openai_api_keys if k and len(k) >= 8]
|
||||
|
||||
|
||||
def _deployment_allows_empty_api_key(deployment: Dict[str, Any]) -> bool:
|
||||
"""Return whether a configured vision deployment is a supported keyless endpoint."""
|
||||
params = deployment.get("litellm_params") or {}
|
||||
if str(params.get("api_key") or "").strip():
|
||||
return False
|
||||
wire_model = str(params.get("model") or "").strip()
|
||||
protocol = wire_model.split("/", 1)[0] if "/" in wire_model else None
|
||||
return channel_allows_empty_api_key(protocol, params.get("api_base"))
|
||||
|
||||
|
||||
def _call_litellm_vision(image_b64: str, mime_type: str, api_key: Optional[str] = None) -> str:
|
||||
"""Extract stock codes from an image using litellm (all providers via OpenAI vision format)."""
|
||||
global litellm
|
||||
@@ -246,14 +277,36 @@ def _call_litellm_vision(image_b64: str, mime_type: str, api_key: Optional[str]
|
||||
if route_has_hermes(getattr(cfg, "llm_model_list", []) or [], model):
|
||||
raise ValueError("Hermes Vision 未验证:VISION_MODEL 不能选择包含 Hermes deployment 的 route。")
|
||||
|
||||
deployments = _matching_vision_deployments(model, cfg)
|
||||
keys = _get_api_keys_for_model(model, cfg)
|
||||
if not keys:
|
||||
key = api_key if api_key and api_key in keys else (random.choice(keys) if keys else None)
|
||||
|
||||
deployment_params: Dict[str, Any] = {}
|
||||
if deployments:
|
||||
deployment = next(
|
||||
(
|
||||
item
|
||||
for item in deployments
|
||||
if str((item.get("litellm_params") or {}).get("api_key") or "").strip() == key
|
||||
),
|
||||
None,
|
||||
)
|
||||
if deployment is None:
|
||||
deployment = next(
|
||||
(item for item in deployments if _deployment_allows_empty_api_key(item)),
|
||||
None,
|
||||
)
|
||||
if deployment is not None:
|
||||
key = None
|
||||
if deployment is not None:
|
||||
deployment_params = dict(deployment.get("litellm_params") or {})
|
||||
if key is None and not deployment_params:
|
||||
raise ValueError(f"No API key found for vision model {model}")
|
||||
key = api_key if api_key and api_key in keys else random.choice(keys)
|
||||
wire_model = str(deployment_params.get("model") or model).strip()
|
||||
|
||||
data_url = f"data:{mime_type};base64,{image_b64}"
|
||||
call_kwargs: dict = {
|
||||
"model": model,
|
||||
"model": wire_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
@@ -264,11 +317,17 @@ def _call_litellm_vision(image_b64: str, mime_type: str, api_key: Optional[str]
|
||||
}
|
||||
],
|
||||
"max_tokens": 1024,
|
||||
"api_key": key,
|
||||
"timeout": VISION_API_TIMEOUT,
|
||||
}
|
||||
effective_api_key = str(deployment_params.get("api_key") or key or "").strip()
|
||||
if effective_api_key:
|
||||
call_kwargs["api_key"] = effective_api_key
|
||||
if deployment_params.get("api_base"):
|
||||
call_kwargs["api_base"] = deployment_params["api_base"]
|
||||
if deployment_params.get("extra_headers"):
|
||||
call_kwargs["extra_headers"] = dict(deployment_params["extra_headers"])
|
||||
# Add api_base and custom headers for OpenAI-compatible providers
|
||||
if not model.startswith("gemini/") and not model.startswith("anthropic/") and not model.startswith("vertex_ai/"):
|
||||
if not deployment_params and not model.startswith("gemini/") and not model.startswith("anthropic/") and not model.startswith("vertex_ai/"):
|
||||
if cfg.openai_base_url:
|
||||
call_kwargs["api_base"] = cfg.openai_base_url
|
||||
if cfg.openai_base_url and "aihubmix.com" in cfg.openai_base_url:
|
||||
|
||||
@@ -7,6 +7,16 @@ import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from src.config import (
|
||||
is_supported_llm_channel_api_surface_value,
|
||||
find_incompatible_llm_channel_models,
|
||||
find_llm_channel_surface_conflicts,
|
||||
normalize_llm_channel_api_surface,
|
||||
normalize_llm_channel_model,
|
||||
resolve_llm_channel_protocol,
|
||||
)
|
||||
from src.llm.hermes import is_reserved_hermes_name
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
_PACKAGE_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_POST_ANALYZERS = ["scorecard"]
|
||||
@@ -409,19 +419,52 @@ def _parse_llm_channels_env() -> list[dict[str, object]]:
|
||||
continue
|
||||
key = name.upper()
|
||||
enabled = _parse_bool_env(f"LLM_{key}_ENABLED", True)
|
||||
base_url = os.getenv(f"LLM_{key}_BASE_URL", "").strip()
|
||||
protocol = os.getenv(f"LLM_{key}_PROTOCOL", "").strip().lower()
|
||||
api_surface_raw = os.getenv(f"LLM_{key}_API_SURFACE", "")
|
||||
api_surface = normalize_llm_channel_api_surface(api_surface_raw)
|
||||
api_keys = (
|
||||
_parse_csv_env(f"LLM_{key}_API_KEYS", [])
|
||||
or _parse_csv_env(f"LLM_{key}_API_KEY", [])
|
||||
)
|
||||
models = _parse_csv_env(f"LLM_{key}_MODELS", [])
|
||||
resolved_protocol = resolve_llm_channel_protocol(
|
||||
protocol,
|
||||
base_url=base_url,
|
||||
models=models,
|
||||
channel_name=name,
|
||||
)
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface_raw):
|
||||
continue
|
||||
effective_protocol = resolved_protocol or "openai"
|
||||
if api_surface == "responses" and effective_protocol != "openai":
|
||||
continue
|
||||
if is_reserved_hermes_name(name) and api_surface == "responses":
|
||||
continue
|
||||
if find_incompatible_llm_channel_models(models, effective_protocol, api_surface, base_url):
|
||||
continue
|
||||
normalized_models = [
|
||||
normalize_llm_channel_model(model, effective_protocol, base_url)
|
||||
for model in models
|
||||
]
|
||||
channels.append({
|
||||
"name": name.lower(),
|
||||
"protocol": os.getenv(f"LLM_{key}_PROTOCOL", "openai").strip().lower(),
|
||||
"base_url": os.getenv(f"LLM_{key}_BASE_URL", "").strip(),
|
||||
"protocol": effective_protocol,
|
||||
"api_surface": api_surface,
|
||||
"base_url": base_url,
|
||||
"api_keys": api_keys,
|
||||
"models": _parse_csv_env(f"LLM_{key}_MODELS", []),
|
||||
"models": normalized_models,
|
||||
"enabled": enabled,
|
||||
})
|
||||
return [channel for channel in channels if channel["enabled"]]
|
||||
enabled_channels = [channel for channel in channels if channel["enabled"]]
|
||||
surface_conflicts = set(find_llm_channel_surface_conflicts(enabled_channels))
|
||||
if not surface_conflicts:
|
||||
return enabled_channels
|
||||
return [
|
||||
channel
|
||||
for channel in enabled_channels
|
||||
if not set(channel.get("models", [])).intersection(surface_conflicts)
|
||||
]
|
||||
|
||||
|
||||
def _resolve_llm_model(channels: list[dict[str, object]]) -> str:
|
||||
|
||||
@@ -9,6 +9,7 @@ import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from src.config import apply_litellm_api_surface
|
||||
from src.llm.errors import call_litellm_with_param_recovery
|
||||
from src.llm.generation_params import apply_litellm_generation_params
|
||||
from src.services.screening.models import Pick
|
||||
@@ -1042,20 +1043,27 @@ def _build_litellm_attempts(
|
||||
channels: list[dict[str, object]],
|
||||
) -> list[dict[str, object]]:
|
||||
attempts = []
|
||||
matched_channel = False
|
||||
for channel in channels:
|
||||
if not _channel_matches_model(channel, model):
|
||||
continue
|
||||
matched_channel = True
|
||||
api_keys = channel.get("api_keys", [])
|
||||
if not isinstance(api_keys, list) or not api_keys:
|
||||
api_keys = [api_key] if api_key else [""]
|
||||
wire_model = apply_litellm_api_surface(
|
||||
model,
|
||||
str(channel.get("api_surface", "") or ""),
|
||||
)
|
||||
for channel_key in api_keys:
|
||||
attempts.append(_completion_kwargs(
|
||||
model,
|
||||
wire_model,
|
||||
api_key=str(channel_key or ""),
|
||||
base_url=str(channel.get("base_url", "") or base_url or ""),
|
||||
))
|
||||
|
||||
attempts.append(_completion_kwargs(model, api_key=api_key, base_url=base_url))
|
||||
if not matched_channel:
|
||||
attempts.append(_completion_kwargs(model, api_key=api_key, base_url=base_url))
|
||||
return _unique_attempts(attempts)
|
||||
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ from urllib.parse import urlparse
|
||||
from fastapi import HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.config import Config, get_configured_llm_models
|
||||
from src.config import Config, get_configured_llm_models, normalize_llm_channel_api_surface
|
||||
from src.services.screening import REFERENCE_PROJECT, REFERENCE_REVISION, __version__ as SCREENING_VERSION
|
||||
from src.services.screening import hotspot as screening_hotspot
|
||||
from src.services.screening.config import Config as ScreeningPipelineConfig
|
||||
@@ -1895,6 +1895,7 @@ def _build_screening_runtime_env(config: Config, *, max_results: Optional[int] =
|
||||
prefix = channel["name"].upper()
|
||||
put(f"LLM_{prefix}_ENABLED", "true")
|
||||
put(f"LLM_{prefix}_PROTOCOL", channel.get("protocol"))
|
||||
put(f"LLM_{prefix}_API_SURFACE", channel.get("api_surface"))
|
||||
put(f"LLM_{prefix}_BASE_URL", channel.get("base_url"))
|
||||
put(f"LLM_{prefix}_API_KEYS", ",".join(channel.get("api_keys") or []))
|
||||
put(f"LLM_{prefix}_MODELS", ",".join(channel.get("models") or []))
|
||||
@@ -3179,6 +3180,7 @@ def _normalize_dsa_llm_channels(config: Config) -> List[Dict[str, Any]]:
|
||||
channel = {
|
||||
"name": name,
|
||||
"protocol": _env_text(raw.get("protocol")),
|
||||
"api_surface": normalize_llm_channel_api_surface(raw.get("api_surface")),
|
||||
"base_url": _env_text(raw.get("base_url")),
|
||||
"api_keys": api_keys,
|
||||
"models": models,
|
||||
|
||||
@@ -20,15 +20,23 @@ import requests
|
||||
from src.config import (
|
||||
ANSPIRE_LLM_BASE_URL_DEFAULT,
|
||||
ANSPIRE_LLM_MODEL_DEFAULT,
|
||||
SUPPORTED_LLM_CHANNEL_API_SURFACES,
|
||||
SUPPORTED_LLM_CHANNEL_PROTOCOLS,
|
||||
Config,
|
||||
_get_litellm_provider,
|
||||
_uses_direct_env_provider,
|
||||
apply_litellm_api_surface,
|
||||
canonicalize_llm_channel_api_surface,
|
||||
canonicalize_llm_channel_protocol,
|
||||
channel_allows_empty_api_key,
|
||||
find_incompatible_llm_channel_models,
|
||||
find_llm_channel_surface_conflicts,
|
||||
get_litellm_model_providers,
|
||||
get_configured_llm_models,
|
||||
is_supported_llm_channel_api_surface_value,
|
||||
normalize_agent_litellm_model,
|
||||
normalize_news_strategy_profile,
|
||||
normalize_llm_channel_api_surface,
|
||||
normalize_llm_channel_model,
|
||||
parse_env_bool,
|
||||
parse_env_int,
|
||||
@@ -160,7 +168,7 @@ class SystemConfigService:
|
||||
"ANSPIRE_API_KEYS",
|
||||
}
|
||||
_GENERATION_BACKEND_STATUS_LLM_CHANNEL_RE = re.compile(
|
||||
r"^LLM_[A-Z0-9_]+_(PROTOCOL|BASE_URL|API_KEY|API_KEYS|MODELS|EXTRA_HEADERS|ENABLED)$"
|
||||
r"^LLM_[A-Z0-9_]+_(PROTOCOL|API_SURFACE|BASE_URL|API_KEY|API_KEYS|MODELS|EXTRA_HEADERS|ENABLED)$"
|
||||
)
|
||||
_AGENT_BACKEND_STATUS_EXACT_KEYS = {
|
||||
"AGENT_BACKEND",
|
||||
@@ -174,7 +182,7 @@ class SystemConfigService:
|
||||
_LLM_CAPABILITY_ORDER: Tuple[str, ...] = ("json", "tools", "stream", "vision")
|
||||
_LLM_STREAM_CHUNK_LIMIT = 8
|
||||
_WEB_SETTINGS_LLM_CHANNEL_SUPPORT_KEY_RE = re.compile(
|
||||
r"^LLM_([A-Z0-9_]+)_(PROTOCOL|BASE_URL|API_KEY|API_KEYS|MODELS|EXTRA_HEADERS|ENABLED)$"
|
||||
r"^LLM_([A-Z0-9_]+)_(PROTOCOL|API_SURFACE|BASE_URL|API_KEY|API_KEYS|MODELS|EXTRA_HEADERS|ENABLED)$"
|
||||
)
|
||||
_LLM_CAPABILITY_PROBE_IMAGE = (
|
||||
"data:image/png;base64,"
|
||||
@@ -492,6 +500,7 @@ class SystemConfigService:
|
||||
"config_version": self._manager.get_config_version(),
|
||||
"mask_token": mask_token,
|
||||
"items": items,
|
||||
"llm_model_providers": sorted(get_litellm_model_providers()),
|
||||
"updated_at": self._manager.get_updated_at(),
|
||||
}
|
||||
|
||||
@@ -1193,6 +1202,7 @@ class SystemConfigService:
|
||||
*,
|
||||
name: str,
|
||||
protocol: str,
|
||||
api_surface: str = "chat_completions",
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
models: Sequence[str],
|
||||
@@ -1205,13 +1215,15 @@ class SystemConfigService:
|
||||
requested_capabilities = self._normalize_llm_capability_checks(capability_checks)
|
||||
raw_models = [str(model).strip() for model in models if str(model).strip()]
|
||||
channel_name = name.strip() or "channel"
|
||||
resolved_api_surface = normalize_llm_channel_api_surface(api_surface)
|
||||
generation_stage = "responses" if resolved_api_surface == "responses" else "chat_completion"
|
||||
resolved_secret, secret_error, redaction_values = self._resolve_hermes_saved_secret(
|
||||
channel_name=channel_name,
|
||||
protocol=protocol,
|
||||
base_url=base_url,
|
||||
submitted_api_key=api_key,
|
||||
use_saved_secret=use_saved_secret,
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
)
|
||||
if resolved_secret is None:
|
||||
result = secret_error
|
||||
@@ -1229,7 +1241,7 @@ class SystemConfigService:
|
||||
secret_error = self._validate_hermes_submitted_secret(
|
||||
api_key=api_key,
|
||||
use_saved_secret=use_saved_secret,
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
capability_checks=requested_capabilities,
|
||||
redaction_values=redaction_values,
|
||||
)
|
||||
@@ -1242,7 +1254,7 @@ class SystemConfigService:
|
||||
success=False,
|
||||
message="Hermes Base URL is invalid",
|
||||
error=str(exc),
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
error_code="invalid_config",
|
||||
retryable=False,
|
||||
details={
|
||||
@@ -1264,6 +1276,7 @@ class SystemConfigService:
|
||||
validation_issues = self._validate_llm_channel_definition(
|
||||
channel_name=channel_name,
|
||||
protocol_value=protocol,
|
||||
api_surface_value=api_surface,
|
||||
base_url_value=base_url,
|
||||
api_key_value=api_key,
|
||||
model_values=raw_models,
|
||||
@@ -1277,7 +1290,7 @@ class SystemConfigService:
|
||||
success=False,
|
||||
message="LLM channel configuration is invalid",
|
||||
error=errors[0]["message"],
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
error_code="invalid_config",
|
||||
retryable=False,
|
||||
details={
|
||||
@@ -1302,12 +1315,13 @@ class SystemConfigService:
|
||||
resolved_model = resolved_models[0]
|
||||
if is_reserved_hermes_name(channel_name):
|
||||
resolved_model = canonicalize_hermes_model_ref(raw_models[0]).wire_model
|
||||
wire_model = apply_litellm_api_surface(resolved_model, resolved_api_surface)
|
||||
api_keys = [segment.strip() for segment in api_key.split(",") if segment.strip()]
|
||||
selected_api_key = api_keys[0] if api_keys else ""
|
||||
redaction_values.update(self._build_redaction_values(selected_api_key))
|
||||
|
||||
call_kwargs: Dict[str, Any] = {
|
||||
"model": resolved_model,
|
||||
"model": wire_model,
|
||||
"messages": [{"role": "user", "content": "Reply with OK"}],
|
||||
"max_tokens": 256, # Increased to allow MiniMax-M3 thinking process + response
|
||||
"timeout": max(5.0, float(timeout_seconds)),
|
||||
@@ -1318,7 +1332,7 @@ class SystemConfigService:
|
||||
call_kwargs["api_base"] = base_url.strip()
|
||||
call_kwargs = apply_litellm_generation_params(
|
||||
call_kwargs,
|
||||
resolved_model,
|
||||
wire_model,
|
||||
self._get_runtime_llm_temperature(),
|
||||
)
|
||||
|
||||
@@ -1354,7 +1368,7 @@ class SystemConfigService:
|
||||
hermes_call_kwargs.pop("api_base", None)
|
||||
response = call_litellm_with_param_recovery(
|
||||
lambda kwargs: litellm.completion(**kwargs),
|
||||
model=resolved_model,
|
||||
model=wire_model,
|
||||
call_kwargs=hermes_call_kwargs,
|
||||
logger=logger,
|
||||
log_label="[Hermes channel test]",
|
||||
@@ -1362,7 +1376,7 @@ class SystemConfigService:
|
||||
else:
|
||||
response = call_litellm_with_param_recovery(
|
||||
lambda kwargs: litellm.completion(**kwargs),
|
||||
model=resolved_model,
|
||||
model=wire_model,
|
||||
call_kwargs=call_kwargs,
|
||||
logger=logger,
|
||||
log_label="[LLM channel test]",
|
||||
@@ -1385,6 +1399,7 @@ class SystemConfigService:
|
||||
details={"response_error": parse_error, "reason": parse_reason},
|
||||
resolved_protocol=resolved_protocol or None,
|
||||
resolved_model=resolved_model,
|
||||
resolved_api_surface=resolved_api_surface,
|
||||
latency_ms=latency_ms,
|
||||
capability_results=self._build_skipped_capability_results(
|
||||
requested_capabilities,
|
||||
@@ -1409,7 +1424,7 @@ class SystemConfigService:
|
||||
elif requested_capabilities:
|
||||
capability_results = self._run_llm_capability_checks(
|
||||
litellm_module=litellm,
|
||||
resolved_model=resolved_model,
|
||||
resolved_model=wire_model,
|
||||
selected_api_key=selected_api_key,
|
||||
base_url=base_url,
|
||||
timeout_seconds=timeout_seconds,
|
||||
@@ -1419,12 +1434,13 @@ class SystemConfigService:
|
||||
success=True,
|
||||
message="LLM channel test succeeded",
|
||||
error=None,
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
error_code=None,
|
||||
retryable=False,
|
||||
details={"response_preview": content[:80]},
|
||||
resolved_protocol=resolved_protocol or None,
|
||||
resolved_model=resolved_model,
|
||||
resolved_api_surface=resolved_api_surface,
|
||||
latency_ms=latency_ms,
|
||||
capability_results=capability_results,
|
||||
redaction_values=redaction_values,
|
||||
@@ -1440,12 +1456,13 @@ class SystemConfigService:
|
||||
success=False,
|
||||
message=diagnostic.message,
|
||||
error=str(exc),
|
||||
stage="chat_completion",
|
||||
stage=generation_stage,
|
||||
error_code=diagnostic.error_code,
|
||||
retryable=diagnostic.retryable,
|
||||
details=self._merge_llm_diagnostic_details({"model": resolved_model}, diagnostic),
|
||||
resolved_protocol=resolved_protocol or None,
|
||||
resolved_model=resolved_model,
|
||||
resolved_api_surface=resolved_api_surface,
|
||||
latency_ms=None,
|
||||
redaction_values=redaction_values,
|
||||
capability_results=self._build_skipped_capability_results(
|
||||
@@ -3432,6 +3449,9 @@ class SystemConfigService:
|
||||
protocol = (effective_map.get(f"{prefix}_PROTOCOL") or "").strip()
|
||||
if name.lower() == "anspire" and not protocol:
|
||||
protocol = "openai"
|
||||
api_surface = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface):
|
||||
continue
|
||||
api_key = (
|
||||
(effective_map.get(f"{prefix}_API_KEYS") or "").strip()
|
||||
or (effective_map.get(f"{prefix}_API_KEY") or "").strip()
|
||||
@@ -3447,6 +3467,8 @@ class SystemConfigService:
|
||||
).strip()
|
||||
]
|
||||
if is_reserved_hermes_name(name):
|
||||
if normalize_llm_channel_api_surface(api_surface) == "responses":
|
||||
continue
|
||||
result = parse_hermes_channel(
|
||||
enabled=True,
|
||||
protocol=protocol or HERMES_DEFAULT_PROTOCOL,
|
||||
@@ -3470,6 +3492,13 @@ class SystemConfigService:
|
||||
)
|
||||
if not raw_models or not resolved_protocol:
|
||||
continue
|
||||
if find_incompatible_llm_channel_models(
|
||||
raw_models,
|
||||
resolved_protocol,
|
||||
api_surface,
|
||||
base_url,
|
||||
):
|
||||
continue
|
||||
if not api_key and not channel_allows_empty_api_key(resolved_protocol, base_url):
|
||||
continue
|
||||
|
||||
@@ -3963,6 +3992,7 @@ class SystemConfigService:
|
||||
retryable: Optional[bool],
|
||||
details: Optional[Dict[str, Any]] = None,
|
||||
resolved_protocol: Optional[str] = None,
|
||||
resolved_api_surface: Optional[str] = None,
|
||||
resolved_model: Optional[str] = None,
|
||||
models: Optional[List[str]] = None,
|
||||
latency_ms: Optional[int] = None,
|
||||
@@ -3981,6 +4011,10 @@ class SystemConfigService:
|
||||
resolved_protocol,
|
||||
redaction_values=redaction_values,
|
||||
) if resolved_protocol is not None else None,
|
||||
"resolved_api_surface": cls._sanitize_llm_error_text(
|
||||
resolved_api_surface,
|
||||
redaction_values=redaction_values,
|
||||
) if resolved_api_surface is not None else None,
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
if resolved_model is not None or models is None:
|
||||
@@ -4621,9 +4655,11 @@ class SystemConfigService:
|
||||
seen_names.add(normalized_upper)
|
||||
normalized_names.append(name)
|
||||
|
||||
validated_channels: List[Dict[str, Any]] = []
|
||||
for name in normalized_names:
|
||||
prefix = f"LLM_{name.upper()}"
|
||||
protocol_value = (effective_map.get(f"{prefix}_PROTOCOL") or "").strip()
|
||||
api_surface_value = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
if name.lower() == "anspire" and not protocol_value:
|
||||
protocol_value = "openai"
|
||||
base_url_value = (effective_map.get(f"{prefix}_BASE_URL") or "").strip()
|
||||
@@ -4654,7 +4690,36 @@ class SystemConfigService:
|
||||
if name.lower() == "anspire" and not (enabled_raw or "").strip():
|
||||
enabled_raw = effective_map.get("ANSPIRE_LLM_ENABLED")
|
||||
enabled = parse_env_bool(enabled_raw, default=True)
|
||||
if not enabled:
|
||||
continue
|
||||
if is_reserved_hermes_name(name):
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface_value):
|
||||
issues.append(
|
||||
{
|
||||
"key": f"{prefix}_API_SURFACE",
|
||||
"code": "invalid_api_surface",
|
||||
"message": (
|
||||
f"Unsupported LLM API surface '{api_surface_value}'. "
|
||||
f"Supported: {', '.join(SUPPORTED_LLM_CHANNEL_API_SURFACES)}"
|
||||
),
|
||||
"severity": "error",
|
||||
"expected": ",".join(SUPPORTED_LLM_CHANNEL_API_SURFACES),
|
||||
"actual": api_surface_value,
|
||||
}
|
||||
)
|
||||
continue
|
||||
if normalize_llm_channel_api_surface(api_surface_value) == "responses":
|
||||
issues.append(
|
||||
{
|
||||
"key": f"{prefix}_API_SURFACE",
|
||||
"code": "hermes_responses_unsupported",
|
||||
"message": "The reserved Hermes channel does not support the Responses API surface",
|
||||
"severity": "error",
|
||||
"expected": "chat_completions",
|
||||
"actual": "responses",
|
||||
}
|
||||
)
|
||||
continue
|
||||
result = parse_hermes_channel(
|
||||
enabled=enabled,
|
||||
protocol=protocol_value or HERMES_DEFAULT_PROTOCOL,
|
||||
@@ -4675,18 +4740,52 @@ class SystemConfigService:
|
||||
"actual": "",
|
||||
}
|
||||
)
|
||||
if result.channel is not None and not result.issues:
|
||||
validated_channels.append(result.channel)
|
||||
continue
|
||||
issues.extend(
|
||||
SystemConfigService._validate_llm_channel_definition(
|
||||
channel_issues = SystemConfigService._validate_llm_channel_definition(
|
||||
channel_name=name,
|
||||
protocol_value=protocol_value,
|
||||
api_surface_value=api_surface_value,
|
||||
base_url_value=base_url_value,
|
||||
api_key_value=api_key_value,
|
||||
model_values=models_value,
|
||||
enabled=enabled,
|
||||
field_prefix=prefix,
|
||||
require_complete=enabled,
|
||||
)
|
||||
issues.extend(channel_issues)
|
||||
if not any(issue.get("severity") == "error" for issue in channel_issues):
|
||||
resolved_protocol = resolve_llm_channel_protocol(
|
||||
protocol_value,
|
||||
base_url=base_url_value,
|
||||
models=models_value,
|
||||
channel_name=name,
|
||||
protocol_value=protocol_value,
|
||||
base_url_value=base_url_value,
|
||||
api_key_value=api_key_value,
|
||||
model_values=models_value,
|
||||
enabled=enabled,
|
||||
field_prefix=prefix,
|
||||
require_complete=enabled,
|
||||
)
|
||||
validated_channels.append(
|
||||
{
|
||||
"name": name.lower(),
|
||||
"protocol": resolved_protocol,
|
||||
"api_surface": normalize_llm_channel_api_surface(api_surface_value),
|
||||
"base_url": base_url_value,
|
||||
"models": models_value,
|
||||
"enabled": True,
|
||||
}
|
||||
)
|
||||
|
||||
for model, surfaces in find_llm_channel_surface_conflicts(validated_channels).items():
|
||||
issues.append(
|
||||
{
|
||||
"key": "LLM_CHANNELS",
|
||||
"code": "mixed_api_surfaces_for_route",
|
||||
"message": (
|
||||
f"LLM route alias '{model}' is declared with multiple API surfaces: "
|
||||
f"{', '.join(surfaces)}"
|
||||
),
|
||||
"severity": "error",
|
||||
"expected": "one API surface per normalized route alias",
|
||||
"actual": ",".join(surfaces),
|
||||
}
|
||||
)
|
||||
|
||||
return issues
|
||||
@@ -4722,6 +4821,7 @@ class SystemConfigService:
|
||||
protocol_value = (effective_map.get(f"{prefix}_PROTOCOL") or "").strip()
|
||||
if name.lower() == "anspire" and not protocol_value:
|
||||
protocol_value = "openai"
|
||||
api_surface_value = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
raw_models = [
|
||||
model.strip()
|
||||
for model in (effective_map.get(f"{prefix}_MODELS") or "").split(",")
|
||||
@@ -4735,6 +4835,11 @@ class SystemConfigService:
|
||||
).strip()
|
||||
]
|
||||
if is_reserved_hermes_name(name):
|
||||
if (
|
||||
not is_supported_llm_channel_api_surface_value(api_surface_value)
|
||||
or normalize_llm_channel_api_surface(api_surface_value) == "responses"
|
||||
):
|
||||
continue
|
||||
result = parse_hermes_channel(
|
||||
enabled=True,
|
||||
protocol=protocol_value or HERMES_DEFAULT_PROTOCOL,
|
||||
@@ -4751,6 +4856,16 @@ class SystemConfigService:
|
||||
models.append(model)
|
||||
continue
|
||||
resolved_protocol = resolve_llm_channel_protocol(protocol_value, base_url=base_url_value, models=raw_models, channel_name=name)
|
||||
if (
|
||||
not is_supported_llm_channel_api_surface_value(api_surface_value)
|
||||
or find_incompatible_llm_channel_models(
|
||||
raw_models,
|
||||
resolved_protocol,
|
||||
api_surface_value,
|
||||
base_url_value,
|
||||
)
|
||||
):
|
||||
continue
|
||||
for model in raw_models:
|
||||
normalized_model = normalize_llm_channel_model(model, resolved_protocol, base_url_value)
|
||||
if not normalized_model or normalized_model in seen:
|
||||
@@ -4779,6 +4894,12 @@ class SystemConfigService:
|
||||
if not enabled:
|
||||
continue
|
||||
|
||||
api_surface_value = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
if (
|
||||
not is_supported_llm_channel_api_surface_value(api_surface_value)
|
||||
or normalize_llm_channel_api_surface(api_surface_value) == "responses"
|
||||
):
|
||||
continue
|
||||
raw_models = SystemConfigService._split_csv(effective_map.get(f"{prefix}_MODELS") or "")
|
||||
result = parse_hermes_channel(
|
||||
enabled=True,
|
||||
@@ -4823,6 +4944,9 @@ class SystemConfigService:
|
||||
protocol_value = (effective_map.get(f"{prefix}_PROTOCOL") or "").strip()
|
||||
if name.lower() == "anspire" and not protocol_value:
|
||||
protocol_value = "openai"
|
||||
api_surface_value = (effective_map.get(f"{prefix}_API_SURFACE") or "").strip()
|
||||
if not is_supported_llm_channel_api_surface_value(api_surface_value):
|
||||
continue
|
||||
raw_models = SystemConfigService._split_csv(effective_map.get(f"{prefix}_MODELS") or "")
|
||||
if name.lower() == "anspire" and not raw_models:
|
||||
raw_models = [
|
||||
@@ -4837,6 +4961,13 @@ class SystemConfigService:
|
||||
models=raw_models,
|
||||
channel_name=name,
|
||||
)
|
||||
if find_incompatible_llm_channel_models(
|
||||
raw_models,
|
||||
resolved_protocol,
|
||||
api_surface_value,
|
||||
base_url_value,
|
||||
):
|
||||
continue
|
||||
for raw_model in raw_models:
|
||||
model = normalize_llm_channel_model(raw_model, resolved_protocol, base_url_value)
|
||||
if model and model not in seen:
|
||||
@@ -5216,6 +5347,7 @@ class SystemConfigService:
|
||||
*,
|
||||
channel_name: str,
|
||||
protocol_value: str,
|
||||
api_surface_value: str,
|
||||
base_url_value: str,
|
||||
api_key_value: str,
|
||||
model_values: Sequence[str],
|
||||
@@ -5237,6 +5369,73 @@ class SystemConfigService:
|
||||
require_base_url=False,
|
||||
)
|
||||
models_key = f"{field_prefix}_MODELS" if field_prefix != "test_channel" else "models"
|
||||
api_surface_key = (
|
||||
f"{field_prefix}_API_SURFACE"
|
||||
if field_prefix != "test_channel"
|
||||
else "api_surface"
|
||||
)
|
||||
canonical_api_surface = canonicalize_llm_channel_api_surface(api_surface_value)
|
||||
resolved_api_surface = normalize_llm_channel_api_surface(api_surface_value)
|
||||
if (
|
||||
canonical_api_surface
|
||||
and canonical_api_surface not in SUPPORTED_LLM_CHANNEL_API_SURFACES
|
||||
):
|
||||
issues.append(
|
||||
{
|
||||
"key": api_surface_key,
|
||||
"code": "invalid_api_surface",
|
||||
"message": (
|
||||
f"Unsupported LLM API surface '{api_surface_value}'. "
|
||||
f"Supported: {', '.join(SUPPORTED_LLM_CHANNEL_API_SURFACES)}"
|
||||
),
|
||||
"severity": "error",
|
||||
"expected": ",".join(SUPPORTED_LLM_CHANNEL_API_SURFACES),
|
||||
"actual": api_surface_value,
|
||||
}
|
||||
)
|
||||
elif resolved_api_surface == "responses" and resolved_protocol != "openai":
|
||||
issues.append(
|
||||
{
|
||||
"key": api_surface_key,
|
||||
"code": "responses_requires_openai_protocol",
|
||||
"message": "Responses API surface currently requires the openai protocol",
|
||||
"severity": "error",
|
||||
"expected": "openai",
|
||||
"actual": resolved_protocol or protocol_value,
|
||||
}
|
||||
)
|
||||
elif resolved_api_surface == "responses" and is_reserved_hermes_name(channel_name):
|
||||
issues.append(
|
||||
{
|
||||
"key": api_surface_key,
|
||||
"code": "hermes_responses_unsupported",
|
||||
"message": "The reserved Hermes channel does not support the Responses API surface",
|
||||
"severity": "error",
|
||||
"expected": "chat_completions",
|
||||
"actual": resolved_api_surface,
|
||||
}
|
||||
)
|
||||
elif resolved_api_surface == "responses":
|
||||
incompatible_models = find_incompatible_llm_channel_models(
|
||||
list(model_values),
|
||||
resolved_protocol,
|
||||
resolved_api_surface,
|
||||
base_url_value,
|
||||
)
|
||||
if incompatible_models:
|
||||
issues.append(
|
||||
{
|
||||
"key": models_key,
|
||||
"code": "responses_requires_openai_model_provider",
|
||||
"message": (
|
||||
"Responses API surface requires every model to use the OpenAI "
|
||||
f"provider route; incompatible: {', '.join(incompatible_models[:3])}"
|
||||
),
|
||||
"severity": "error",
|
||||
"expected": "openai/<model> or an unprefixed OpenAI-compatible model ID",
|
||||
"actual": ", ".join(incompatible_models[:3]),
|
||||
}
|
||||
)
|
||||
|
||||
if not model_values:
|
||||
issues.append(
|
||||
|
||||
Reference in New Issue
Block a user