From 4aad40cdd43d8809b09687c9c07372db29560f36 Mon Sep 17 00:00:00 2001 From: mumu <42829555+ZhuLinsen@users.noreply.github.com> Date: Sat, 16 May 2026 21:31:28 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A2=9E=E5=BC=BA=20LLM=20=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E9=80=82=E9=85=8D=E5=B1=82=20(#1317)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: add llm generation parameter adaptation * fix(review-feedback-1317): 补充官方来源链接或明确写成基于当前兼容测试的保守策略，并说明验证环境 * fix(review-feedback-1317): 补充官方文档/公告链接、当前依赖/运行时验证依据，以及 provider override 的预期扩展方式 * feat: 增强 LLM 参数错误自愈 (#1318) * fix: add llm generation parameter adaptation * fix: add llm parameter error recovery * fix(review-feedback-1318): Scope recovery cache to the endpoint and making recovery caching * fix(review-feedback-1318): Avoid forcing 1.0 for all default-only temperature errors * fix(review-feedback-1318): 基于目标分支解决冲突，并确保解决后 diff 仍只包含本 PR 声称的 LLM 参数自愈增量 * fix(review-feedback-1318): Scope legacy Router recoveries to their endpoint * fix(review-feedback-1318): 解决冲突并基于解决后的最终 diff 重新确认测试结果 * fix(review-feedback-1318): 基于目标 base 解决冲突，再重新确认 diff 与 CI * fix(review-feedback-1318): 解决冲突后重新确认 diff 与 CI * fix(review-feedback-1318): 补充覆盖 Analyzer legacy multi-key Router 的回归测试 --- .env.example | 12 +- docs/CHANGELOG.md | 4 + docs/LLM_CONFIG_GUIDE.md | 18 +- docs/LLM_CONFIG_GUIDE_EN.md | 18 +- src/agent/llm_adapter.py | 66 ++- src/analyzer.py | 189 +++++++-- src/config.py | 123 +----- src/llm/__init__.py | 2 + src/llm/errors.py | 151 +++++++ src/llm/generation_params.py | 439 ++++++++++++++++++++ src/services/system_config_service.py | 26 +- tests/test_agent_pipeline.py | 166 ++++++++ tests/test_llm_channel_config.py | 36 ++ tests/test_llm_param_recovery.py | 202 +++++++++ tests/test_market_analyzer_generate_text.py | 203 +++++++++ tests/test_system_config_service.py | 94 +++++ 16 files changed, 1554 insertions(+), 195 deletions(-) create mode 100644 src/llm/__init__.py create mode 100644 src/llm/errors.py create mode 100644 src/llm/generation_params.py create mode 100644 tests/test_llm_param_recovery.py diff --git a/.env.example b/.env.example index 145ef4ac3..482f0b9eb 100644 --- a/.env.example +++ b/.env.example @@ -96,14 +96,18 @@ GEMINI_API_KEY= # LLM_OLLAMA_MODELS=qwen3:8b # 采样温度（0.0-2.0，默认 0.7；0 确定性最高，2 随机性最高） -# Kimi K2.6 兼容来源： +# 严格 temperature 兼容来源： # - Moonshot API / 模型文档：https://platform.kimi.ai/docs/guide/kimi-k2-6-quickstart # - Moonshot 官方模型卡（评测默认 temperature = 1.0）：https://huggingface.co/moonshotai/Kimi-K2.6 +# - OpenAI Chat Completions 规范：https://platform.openai.com/docs/api-reference/chat/create # - LiteLLM OpenAI-Compatible 规范：https://docs.litellm.ai/docs/providers/openai_compatible # 当前仓库运行时依赖约束：litellm>=1.80.10,!=1.82.7,!=1.82.8,<2.0.0（显式排除 PyPI 事故版本，见 requirements.txt）。 -# 因此 kimi-k2.6 会自动改用 1.0，避免 API 拒绝请求；其他模型和 fallback 仍使用你配置的 LLM_TEMPERATURE。 -# Web 设置页 / 桌面端导入不会静默清空或改写 LLM_TEMPERATURE；只在真正发请求前按 Kimi 要求临时归一化。 -# 如果主模型切回非 Kimi，原本配置的温度会自动恢复；最小回滚方式是回退本次 Kimi 固定温度改动。 +# 因此 kimi-k2.6 会自动改用 1.0/0.6；GPT-5 / o 系列等默认温度模型会省略 temperature，避免 API 拒绝请求。 +# 若兼容平台返回明确的参数不支持错误，运行时会在当前请求内修正参数并重试一次；成功策略只做进程内缓存。 +# top_p、presence_penalty、frequency_penalty、seed 若返回“不支持参数”，同样会触发本次请求级别的修正与重试（不改写 LLM_TEMPERATURE）。 +# 其他模型和 fallback 仍使用你配置的 LLM_TEMPERATURE。 +# Web 设置页 / 桌面端导入不会静默清空或改写 LLM_TEMPERATURE；只在真正发请求前临时适配模型参数。 +# 如果主模型切回普通模型，原本配置的温度会自动恢复；最小回滚方式是回退本次 LLM 参数适配改动。 # LLM_TEMPERATURE=0.7 # --- 多渠道配置（可选，也可在 Web 设置页配置）--- diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6a9f7e9da..1901b9ef3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). +- [修复] 抽出 LiteLLM 生成参数适配层，对严格 temperature 模型按请求临时固定或省略参数，避免 GPT-5 / o 系列与 Kimi K2.6 拒绝默认温度请求。 +- [改进] LiteLLM 参数错误支持一次请求内自动修正重试，并在成功后进程内缓存策略，降低新模型参数兼容问题的人工配置成本。 +- [文档] 补充 Issue #1316 参数自愈改动的外部兼容依据、运行时配置清理边界与回滚证据；并在 `tests/test_system_config_service.py` 增加清理路径下 `LLM_TEMPERATURE` 保持不变的回归用例。 +- [文档] 补充严格 temperature 兼容语义的官方来源、运行时依赖约束与 `LLM_TEMPERATURE` 回退/不回写路径说明。 - [修复] 统一 Windows 桌面安装包与自动更新元数据文件名，避免 Release 中出现重复安装包并阻断 `latest.yml` 指向不存在附件。 - [修复] 桌面端启动 WebUI 时为入口页增加 no-cache 响应头和版本化 cache-busting URL，避免安装新版后 Electron 继续复用旧 WebUI 缓存。 diff --git a/docs/LLM_CONFIG_GUIDE.md b/docs/LLM_CONFIG_GUIDE.md index 9c61771a8..d075d6c89 100644 --- a/docs/LLM_CONFIG_GUIDE.md +++ b/docs/LLM_CONFIG_GUIDE.md @@ -199,17 +199,20 @@ LITELLM_MODEL=ollama/qwen3:8b - 如果当前环境没有任何有效 Agent 模型链路，问股页面会继续按失败语义返回，并直接展示后端真实配置诊断；补齐任一有效模型来源后即可恢复，无需额外执行配置迁移脚本。 - 推荐的新配置方式仍然是显式设置 `LITELLM_MODEL` / `AGENT_LITELLM_MODEL` 或使用 `LLM_CHANNELS`；legacy provider keys 目前保留为兼容回退路径，方便旧 `.env`、本地 macOS 开发环境和历史部署平滑继续运行。 -### Kimi K2.6 固定 temperature 兼容说明 +### 严格 temperature 模型兼容说明 - Moonshot 官方说明 Kimi API 兼容 OpenAI 接口，Base URL 使用 `https://api.moonshot.ai/v1`： - LiteLLM 官方要求 OpenAI Compatible 渠道模型名使用 `openai/` 前缀： - Moonshot 官方兼容性文档区分两种固定值：**thinking 模式固定 `1.0`，non-thinking 模式固定 `0.6`**；传其它值会被接口拒绝： +- OpenAI Chat Completions 规范中 `temperature` 是可选参数；对 GPT-5 / o 系列等只接受默认温度的模型，本项目会在请求层省略 `temperature`，让服务端使用默认值，而不是改写你的 `LLM_TEMPERATURE`： - 当前仓库的运行时依赖约束是 `litellm>=1.80.10,!=1.82.7,!=1.82.8,<2.0.0`（见 `requirements.txt`）；本次兼容逻辑按该约束回归验证了主分析、大盘复盘、Agent 直连 LiteLLM，以及系统设置页的渠道连通性测试。 - 因此本项目会在请求发出前按**实际请求模式**归一化 `kimi-k2.6` 及其 `kimi-k2.6-*` 变体：默认 / thinking 路径使用 `temperature=1.0`；如果你的 LiteLLM YAML 路由别名里显式写了 `litellm_params.extra_body.thinking.type: disabled`（或等价 non-thinking 配置），则自动切到 `temperature=0.6`。你在 `.env` 或 Web 设置里保存的 `LLM_TEMPERATURE` 不会被改写。 -- `SystemConfigService` 在 Web 设置保存 / 桌面端 `.env` 导入时只更新你提交的 key，不会因为切到 Kimi 静默清空、迁移或重写已有 `LLM_TEMPERATURE`；渠道测试请求里临时使用的 `1.0/0.6` 也不会回写到配置文件。 -- 非 Kimi 主模型、非 Kimi fallback 以及切回普通模型后的请求，仍继续使用你配置的温度；也就是说旧配置无需迁移，切换模型即可自动恢复原行为。 +- 如果兼容平台对未收录的新模型返回明确的参数错误（例如 `temperature` 不支持、只能使用默认 `1.0`、`top_p` 不支持），运行时会对**当前请求**做一次参数修正并重试；只有重试成功后才把该策略缓存在当前进程内。该缓存不会写回 `.env`，服务重启后会重新按配置与适配规则判断。 +- 对已经产生部分内容的流式响应，系统不会在半截输出后切换参数；仍沿用原有“同模型非流式重试 / fallback 模型”的稳定路径，避免拼接出不一致的回答。 +- `SystemConfigService` 在 Web 设置保存 / 桌面端 `.env` 导入时只更新你提交的 key，不会因为切到严格 temperature 模型静默清空、迁移或重写已有 `LLM_TEMPERATURE`；渠道测试请求里的临时参数策略也不会回写到配置文件。 +- 非严格主模型、非严格 fallback 以及切回普通模型后的请求，仍继续使用你配置的温度；也就是说旧配置无需迁移，切换模型即可自动恢复原行为。 - 本仓库兼容性回归覆盖见：`tests/test_llm_channel_config.py`、`tests/test_market_analyzer_generate_text.py`、`tests/test_agent_pipeline.py`、`tests/test_system_config_service.py`。 -- 最小回滚方式：直接回退本次 Kimi 固定温度相关改动，无需单独迁移已有 `LLM_TEMPERATURE` 配置。 +- 最小回滚方式：直接回退本次 LLM 参数适配相关改动，无需单独迁移已有 `LLM_TEMPERATURE` 配置。 ### 兼容性与回退复核清单（按 PR 审核口径） @@ -224,6 +227,12 @@ LITELLM_MODEL=ollama/qwen3:8b > **致命避坑说明**：如果你启用了 `LLM_CHANNELS`，那么你直接写在外面的 `DEEPSEEK_API_KEY` 或 `OPENAI_API_KEY` 将**全部失效（系统一律无视）**！二者**选其一即可**，千万不要既写了新手模式又写了渠道模式结果产生冲突。 > **Docker 注意**：如果你在 `docker compose environment:` 或 `docker run -e` 中显式传入 `LITELLM_MODEL`、`LLM_CHANNELS`、`LLM_DEEPSEEK_MODELS` 等变量，容器重启后这些环境变量会覆盖 Web 设置页写入的 `.env`，需要同步修改部署配置。 +### 兼容依据与回退审计说明（本次 PR 适配说明） + +- 官方与运行时兼容依据采用两层：第一层为官方接口语义（LiteLLM OpenAI-compatible 路由、OpenAI Chat Completions、Moonshot/Kimi 文档与官方模型说明）；第二层为本仓库当前运行时语义（`litellm>=1.80.10,!=1.82.7,!=1.82.8,<2.0.0`）下的实际错误归类。 +- 本次兼容恢复只使用“本地运行时错误归类 + 单请求修正重试 + 进程内缓存”策略，不写入 `.env`、不做配置迁移，仅在执行路径上动态规避不支持参数（`temperature`、`top_p`、`presence_penalty`、`frequency_penalty`、`seed`）。若要回退，不需要额外迁移命令，恢复旧值即可。 +- 回归与证据：`tests/test_llm_param_recovery.py`、`tests/test_system_config_service.py`、`tests/test_llm_channel_config.py`、`tests/test_system_config_api.py`、`tests/test_market_analyzer_generate_text.py`、`tests/test_agent_pipeline.py`；桌面导入与运行时清理回退另有 `test_import_desktop_env_restores_runtime_models_after_cleanup` 直接覆盖。 + --- ## 方式三：YAML 高级配置（适合老手自定义） @@ -318,6 +327,7 @@ VISION_PROVIDER_PRIORITY=gemini,anthropic,openai | **我写了好几家的Key，为什么死活只有一个生效？修改还没用？** | 你把 **极简模式** 和 **渠道模式** 混着写了！ | 想好一条路走到黑——只要简单就删掉 `LLM_CHANNELS` 开头的；想要丰富备用切换就要全部转投到 `LLM_CHANNELS` 下的编制里。 | | **错误码报 400 或 401 或 Invalid API Key** | API Key 填错、少复制了一截、账号充值没到账、或者模型名字敲错（极度常见）。 | 1. 检查复制的 Key 前后是否有误填空格。
2. 检查 Base URL 最后是不是少了一个 `/v1`。
3. 检查模型名是否少写了 `openai/` 之类的前缀！ | | **Kimi K2.6 报 `invalid temperature`（可能提示只允许 `1.0` 或 `0.6`）** | 该模型按 thinking / non-thinking 模式要求不同固定 temperature；旧配置或调用入口可能还在传 `0.7`。 | 升级后系统会对 `kimi-k2.6` 默认 / thinking 请求自动使用 `temperature=1.0`；如果你在 LiteLLM YAML 路由里显式关闭 thinking，则自动改用 `0.6`。模型名建议写成 `openai/kimi-k2.6` 并配合 Moonshot / 聚合平台的 OpenAI 兼容 Base URL 与 API Key。非 Kimi fallback 仍会继续使用你配置的 `LLM_TEMPERATURE`。 | +| **GPT-5 / o 系列报 `temperature` 不支持或只允许默认值** | 这类模型只接受服务端默认采样参数，但旧调用入口会显式传 `0.7`。 | 升级后请求层会省略 `temperature`，让服务端使用默认值；`.env` / Web 设置中的 `LLM_TEMPERATURE` 不会被改写，切回普通模型后仍按原值发送。 | | **转圈转不停，最后报 Timeout / ConnectionRefused 等** | 1. 在国内使用国外原版（像 Google、OpenAI），没开代理被墙了。
2. 你买的云服务器压根不能出境。 | 非常推荐使用**国内官方**（如DeepSeek、阿里）或者各种**兼容 OpenAI 的聚合中转接口**。因为中转站把网络问题帮你解决好了。 | | **Ollama 报 404、`Could not get model info` 或 `api/generate/api/show`** | 误用 `OPENAI_BASE_URL` 配置 Ollama，系统会错误拼接 URL | 改用 `OLLAMA_API_BASE=http://localhost:11434` 或渠道模式（`LLM_CHANNELS=ollama` + `LLM_OLLAMA_BASE_URL`） | diff --git a/docs/LLM_CONFIG_GUIDE_EN.md b/docs/LLM_CONFIG_GUIDE_EN.md index 41aeee42d..6cfdadfd7 100644 --- a/docs/LLM_CONFIG_GUIDE_EN.md +++ b/docs/LLM_CONFIG_GUIDE_EN.md @@ -192,21 +192,30 @@ LITELLM_MODEL=ollama/qwen3:8b - If the current environment has no valid Agent model path at all, the ask-stock page still returns a failure and now surfaces the backend's real configuration diagnosis. As soon as you restore any valid model source, the flow recovers without running any migration step. - The recommended forward path is still to configure `LITELLM_MODEL` / `AGENT_LITELLM_MODEL` explicitly or move to `LLM_CHANNELS`; legacy provider keys remain a compatibility fallback for older `.env` files, local macOS development, and existing deployments. -### Kimi K2.6 Fixed-Temperature Compatibility Notes +### Strict Temperature Model Compatibility Notes - Moonshot officially documents Kimi as an OpenAI-compatible API, with `https://api.moonshot.ai/v1` as the base URL: - LiteLLM officially requires the `openai/` prefix for OpenAI-compatible model routing: - Moonshot's compatibility docs distinguish two fixed values: **thinking mode must use `1.0`, while non-thinking mode must use `0.6`**; other values are rejected by the API: +- The OpenAI Chat Completions API treats `temperature` as optional. For GPT-5 / o-series style models that only accept the provider default temperature, this project omits `temperature` at request time instead of rewriting your saved `LLM_TEMPERATURE`: - The current runtime dependency constraint in this repository is `litellm>=1.80.10,!=1.82.7,!=1.82.8,<2.0.0` (see `requirements.txt`); this compatibility fix is regression-covered under that constraint across the main analyzer, market review, direct Agent LiteLLM calls, and the system-settings channel connectivity test path. - This repository therefore normalizes `kimi-k2.6` and `kimi-k2.6-*` right before dispatch based on the **actual request mode**: default / thinking requests use `temperature=1.0`; if your LiteLLM YAML route alias explicitly sets `litellm_params.extra_body.thinking.type: disabled` (or an equivalent non-thinking override), it automatically switches to `temperature=0.6`. Your saved `LLM_TEMPERATURE` value in `.env` or the Web settings is not rewritten. -- `SystemConfigService` only updates keys that you actually submit when saving from the Web settings page or importing a desktop `.env`; switching to Kimi does not silently clear, migrate, or rewrite an existing `LLM_TEMPERATURE`. The temporary `1.0/0.6` used for Kimi channel tests is request-scoped and is not persisted back into the config file. -- Non-Kimi primary models, non-Kimi fallbacks, and any request after switching away from Kimi still use your configured temperature. Existing configs do not need migration; changing the model restores the original behavior automatically. +- If a compatible platform returns an explicit parameter error for a not-yet-profiled model, such as unsupported `temperature`, default-only `1.0`, or unsupported `top_p`, the runtime repairs the **current request** and retries once. The strategy is cached only in the current process after the retry succeeds; it is never written back to `.env`, and a service restart re-evaluates the configured rules normally. +- For streaming responses that already produced partial content, the runtime does not switch parameters mid-output. It keeps the existing same-model non-stream retry / fallback-model path to avoid stitching inconsistent answers together. +- `SystemConfigService` only updates keys that you actually submit when saving from the Web settings page or importing a desktop `.env`; switching to a strict-temperature model does not silently clear, migrate, or rewrite an existing `LLM_TEMPERATURE`. Temporary request-time parameter strategies are not persisted back into the config file. +- Non-strict primary models, non-strict fallbacks, and any request after switching back to a regular model still use your configured temperature. Existing configs do not need migration; changing the model restores the original behavior automatically. - Repository-side compatibility coverage lives in `tests/test_llm_channel_config.py`, `tests/test_market_analyzer_generate_text.py`, `tests/test_agent_pipeline.py`, and `tests/test_system_config_service.py`. -- Minimal rollback: revert only the Kimi fixed-temperature change set; no separate `LLM_TEMPERATURE` migration is required. +- Minimal rollback: revert only the LLM generation-parameter adaptation change set; no separate `LLM_TEMPERATURE` migration is required. > **Critical Warning**: If you enable `LLM_CHANNELS`, any standard `DEEPSEEK_API_KEY` or `OPENAI_API_KEY` declared independently will be **completely ignored**. **Use only one mode** to prevent configuration conflicts. > **Docker note**: If `LITELLM_MODEL`, `LLM_CHANNELS`, `LLM_DEEPSEEK_MODELS`, or related variables are explicitly passed through `docker compose environment:` or `docker run -e`, they will override the `.env` written by the Web settings page after a container restart. Update the deployment environment at the same time. +### Compatibility evidence and rollback audit notes (for this recovery change) + +- Compatibility is validated in two layers: first-party provider/API contract references (LiteLLM OpenAI-compatible routing, OpenAI Chat Completions, Moonshot/Kimi docs and model notes), and second the current runtime implementation in this repository under `litellm>=1.80.10,!=1.82.7,!=1.82.8,<2.0.0`. +- This recovery path is runtime-only and intentionally local: exception classification + one in-request repair retry + in-process cache. It does not rewrite `.env`, migrate saved config keys, or alter legacy values; it only omits/adjusts request parameters (`temperature`, `top_p`, `presence_penalty`, `frequency_penalty`, `seed`) for the current call. Rolling back requires no migration; restore previous settings and model/provider selection. +- Regression evidence for this path is in `tests/test_llm_param_recovery.py`, `tests/test_system_config_service.py`, `tests/test_llm_channel_config.py`, `tests/test_system_config_api.py`, `tests/test_market_analyzer_generate_text.py`, `tests/test_agent_pipeline.py`; desktop backup import restore is directly covered by `test_import_desktop_env_restores_runtime_models_after_cleanup`. + --- ## Method 3: Advanced YAML Config (Expert Setup) @@ -285,6 +294,7 @@ Afraid you got the config wrong? Type the following commands in your terminal to | **I added multiple provider Keys, why is only one working?** | You mixed the **Simple Mode** and **Channels Mode**! | Choose one path. For simple setups, delete anything starting with `LLM_CHANNELS`. To use multi-model fallbacks, migrate all your Keys into the `LLM_CHANNELS` setup. | | **Returns 400, 401, or Invalid API Key** | The API Key is wrong, copied incompletely, account lacks credits, or you mistyped the model name (extremely common). | 1. Ensure there are no spaces at the start/end of your Key.
2. Ensure your Base URL ends with `/v1`.
3. Check if you forgot the `openai/` prefix on the model name! | | **Kimi K2.6 returns `invalid temperature` (it may say only `1.0` or `0.6` is allowed)** | The model requires different fixed temperatures for thinking vs non-thinking mode, while older config or call paths may still pass `0.7`. | After this fix, default / thinking `kimi-k2.6` requests automatically use `temperature=1.0`; if you explicitly disable thinking in a LiteLLM YAML route, the request automatically uses `0.6` instead. Prefer `openai/kimi-k2.6` with your Moonshot or relay OpenAI-compatible Base URL and API key. Non-Kimi fallbacks still keep your configured `LLM_TEMPERATURE`. | +| **GPT-5 / o-series returns that `temperature` is unsupported or only the default is allowed** | These models only accept the provider default sampling parameters, while older call paths may still send `0.7`. | The request layer now omits `temperature` so the provider default is used. Your `.env` / Web `LLM_TEMPERATURE` is not rewritten, and regular models keep using it after you switch back. | | **Spins endlessly, eventually hits Timeout/ConnectionRefused** | You are using restricted APIs (like Google/OpenAI) in a blocked region without a proxy, or your cloud server lacks external internet access. | Highly recommend using **official regional APIs** (like DeepSeek) or **OpenAI-compatible relay platforms**. Third-party platforms bypass these network constraints. | | **Ollama returns 404, `Could not get model info`, or `api/generate/api/show`** | Using `OPENAI_BASE_URL` for Ollama makes the system concatenate URLs incorrectly | Use `OLLAMA_API_BASE=http://localhost:11434` or channel mode (`LLM_CHANNELS=ollama` + `LLM_OLLAMA_BASE_URL`) instead | diff --git a/src/agent/llm_adapter.py b/src/agent/llm_adapter.py index 568684194..42646cb0e 100644 --- a/src/agent/llm_adapter.py +++ b/src/agent/llm_adapter.py @@ -23,8 +23,9 @@ from src.config import ( get_configured_llm_models, get_effective_agent_models_to_try, get_effective_agent_primary_model, - normalize_litellm_temperature, ) +from src.llm.errors import call_litellm_with_param_recovery +from src.llm.generation_params import apply_litellm_generation_params logger = logging.getLogger(__name__) @@ -156,6 +157,7 @@ class LLMToolAdapter: config = config or get_config() self._config = config self._router = None # litellm Router (multi-key primary model) + self._legacy_router_model_list: List[Dict[str, Any]] = [] self._litellm_available = False self._register_custom_model_pricing() self._init_litellm() @@ -186,6 +188,7 @@ class LLMToolAdapter: def _init_litellm(self) -> None: """Initialize litellm Router from channels / YAML / legacy keys.""" config = self._config + self._legacy_router_model_list = [] litellm_model = get_effective_agent_primary_model(config) if not litellm_model: logger.warning("Agent LLM: no effective primary model configured") @@ -232,6 +235,7 @@ class LLMToolAdapter: } for k in keys ] + self._legacy_router_model_list = legacy_model_list self._router = Router( model_list=legacy_model_list, routing_strategy="simple-shuffle", @@ -404,12 +408,6 @@ class LLMToolAdapter: call_kwargs: Dict[str, Any] = { "model": model, "messages": openai_messages, - "temperature": normalize_litellm_temperature( - model, - self._get_temperature() if temperature is None else temperature, - model_list=self._config.llm_model_list, - request_overrides={"extra_body": extra} if extra else None, - ), } if max_tokens is not None: call_kwargs["max_tokens"] = max_tokens @@ -426,21 +424,53 @@ class LLMToolAdapter: use_channel_router = self._has_channel_config() _router_model_names = set(get_configured_llm_models(self._config.llm_model_list)) agent_primary_model = get_effective_agent_primary_model(self._config) - if use_channel_router and self._router and model in _router_model_names: - # Channel / YAML path: Router manages all models in its model_list - response = self._router.completion(**call_kwargs) - elif self._router and model == agent_primary_model and not use_channel_router: - # Legacy path: Router for primary model multi-key - response = self._router.completion(**call_kwargs) - else: - # Legacy/direct-env path: direct call (also handles direct-env - # providers like groq/ or bedrock/ that are not in the Router - # model_list even when channel mode is active) + uses_router = ( + bool(use_channel_router and self._router and model in _router_model_names) + or bool(self._router and model == agent_primary_model and not use_channel_router) + ) + recovery_model_list = self._config.llm_model_list + if self._router and model == agent_primary_model and not use_channel_router: + recovery_model_list = self._legacy_router_model_list or self._config.llm_model_list + if not uses_router: keys = get_api_keys_for_model(model, self._config) if keys: call_kwargs["api_key"] = keys[0] call_kwargs.update(extra_litellm_params(model, self._config)) - response = litellm.completion(**call_kwargs) + call_kwargs = apply_litellm_generation_params( + call_kwargs, + model, + self._get_temperature() if temperature is None else temperature, + model_list=recovery_model_list, + ) + if use_channel_router and self._router and model in _router_model_names: + # Channel / YAML path: Router manages all models in its model_list + response = call_litellm_with_param_recovery( + lambda kwargs: self._router.completion(**kwargs), + model=model, + call_kwargs=call_kwargs, + model_list=recovery_model_list, + logger=logger, + ) + elif self._router and model == agent_primary_model and not use_channel_router: + # Legacy path: Router for primary model multi-key + response = call_litellm_with_param_recovery( + lambda kwargs: self._router.completion(**kwargs), + model=model, + call_kwargs=call_kwargs, + model_list=recovery_model_list, + logger=logger, + ) + else: + # Legacy/direct-env path: direct call (also handles direct-env + # providers like groq/ or bedrock/ that are not in the Router + # model_list even when channel mode is active) + response = call_litellm_with_param_recovery( + lambda kwargs: litellm.completion(**kwargs), + model=model, + call_kwargs=call_kwargs, + model_list=self._config.llm_model_list, + logger=logger, + ) return self._parse_litellm_response(response, model) diff --git a/src/analyzer.py b/src/analyzer.py index 13fdaa741..15e0274b7 100644 --- a/src/analyzer.py +++ b/src/analyzer.py @@ -30,9 +30,10 @@ from src.config import ( get_api_keys_for_model, get_config, get_configured_llm_models, - normalize_litellm_temperature, resolve_news_window_days, ) +from src.llm.generation_params import apply_litellm_generation_params +from src.llm.errors import call_litellm_with_param_recovery from src.storage import persist_llm_usage from src.data.stock_mapping import STOCK_NAME_MAP from src.report_language import ( @@ -1810,6 +1811,7 @@ class GeminiAnalyzer: self._use_legacy_default_prompt_override = use_legacy_default_prompt self._resolved_prompt_state: Optional[Dict[str, Any]] = None self._router = None + self._legacy_router_model_list: List[Dict[str, Any]] = [] self._litellm_available = False self._init_litellm() if not self._litellm_available: @@ -1908,6 +1910,47 @@ class GeminiAnalyzer: e.get('model_name', '').startswith('__legacy_') for e in config.llm_model_list ) + @staticmethod + def _legacy_router_provider_alias(model: str) -> str: + provider = model.split("/", 1)[0] if "/" in model else "openai" + return f"__legacy_{provider}__" + + @staticmethod + def _build_legacy_router_model_list_from_config( + model: str, + model_list: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """Build legacy-router candidates from configured legacy llm_model_list entries.""" + if not model: + return [] + target_model = model + target_legacy_alias = GeminiAnalyzer._legacy_router_provider_alias(model) + legacy_entries: List[Dict[str, Any]] = [] + for entry in model_list or []: + if not isinstance(entry, dict): + continue + model_name = str(entry.get("model_name") or "").strip() + if model_name != target_legacy_alias: + continue + + params = entry.get("litellm_params") + if not isinstance(params, dict): + continue + + api_key = str(params.get("api_key") or "").strip() + if not api_key or len(api_key) < 8: + continue + + deployed_params = dict(params) + deployed_params["model"] = target_model + deployed_params["api_key"] = api_key + legacy_entries.append({ + "model_name": target_model, + "litellm_params": deployed_params, + }) + + return legacy_entries + def _init_litellm(self) -> None: """Initialize litellm Router from channels / YAML / legacy keys.""" config = self._get_runtime_config() @@ -1921,27 +1964,34 @@ class GeminiAnalyzer: # --- Channel / YAML path: build Router from pre-built model_list --- if self._has_channel_config(config): model_list = config.llm_model_list - self._router = Router( - model_list=model_list, - routing_strategy="simple-shuffle", - num_retries=2, - ) - unique_models = list(dict.fromkeys( - e['litellm_params']['model'] for e in model_list - )) - logger.info( - f"Analyzer LLM: Router initialized from channels/YAML — " - f"{len(model_list)} deployment(s), models: {unique_models}" - ) - return + try: + self._router = Router( + model_list=model_list, + routing_strategy="simple-shuffle", + num_retries=2, + ) + except TypeError: + logger.debug("Analyzer LLM: Router constructor signature not compatible; fallback to direct mode") + self._router = None + else: + unique_models = list(dict.fromkeys( + e['litellm_params']['model'] for e in model_list + )) + logger.info( + f"Analyzer LLM: Router initialized from channels/YAML — " + f"{len(model_list)} deployment(s), models: {unique_models}" + ) + return # --- Legacy path: build Router for multi-key, or use single key --- keys = get_api_keys_for_model(litellm_model, config) - - if len(keys) > 1: - # Build legacy Router for primary model multi-key load-balancing + legacy_model_list = self._build_legacy_router_model_list_from_config( + litellm_model, + config.llm_model_list, + ) + if len(legacy_model_list) <= 1 and keys: extra_params = extra_litellm_params(litellm_model, config) - legacy_model_list = [ + configured_model_list = [ { "model_name": litellm_model, "litellm_params": { @@ -1952,16 +2002,30 @@ class GeminiAnalyzer: } for k in keys ] - self._router = Router( - model_list=legacy_model_list, - routing_strategy="simple-shuffle", - num_retries=2, - ) - logger.info( - f"Analyzer LLM: Legacy Router initialized with {len(keys)} keys " - f"for {litellm_model}" - ) - elif keys: + if not legacy_model_list: + legacy_model_list = configured_model_list + elif len(legacy_model_list) < len(configured_model_list): + legacy_model_list = configured_model_list + + if len(legacy_model_list) > 1: + self._legacy_router_model_list = legacy_model_list + try: + self._router = Router( + model_list=legacy_model_list, + routing_strategy="simple-shuffle", + num_retries=2, + ) + except TypeError: + logger.debug("Analyzer LLM: Legacy Router constructor signature not compatible; using legacy model_list fallback") + self._router = None + else: + logger.info( + f"Analyzer LLM: Legacy Router initialized with {len(legacy_model_list)} keys " + f"for {litellm_model}" + ) + return + + if keys: logger.info(f"Analyzer LLM: litellm initialized (model={litellm_model})") else: logger.info( @@ -2206,6 +2270,11 @@ class GeminiAnalyzer: effective_system_prompt = system_prompt or self.TEXT_SYSTEM_PROMPT router_model_names = set(get_configured_llm_models(config.llm_model_list)) for model in models_to_try: + recovery_model_list = config.llm_model_list + legacy_router_model_list = getattr(self, "_legacy_router_model_list", None) or [] + if legacy_router_model_list and model == config.litellm_model and not use_channel_router: + recovery_model_list = legacy_router_model_list + try: model_short = model.split("/")[-1] if "/" in model else model extra = get_thinking_extra_body(model_short) @@ -2215,28 +2284,50 @@ class GeminiAnalyzer: {"role": "system", "content": effective_system_prompt}, {"role": "user", "content": prompt}, ], - "temperature": normalize_litellm_temperature( - model, - requested_temperature, - model_list=config.llm_model_list, - request_overrides={"extra_body": extra} if extra else None, - ), "max_tokens": max_tokens, } if extra: call_kwargs["extra_body"] = extra + uses_router = ( + (use_channel_router and self._router and model in router_model_names) + or (self._router and model == config.litellm_model and not use_channel_router) + ) + if not uses_router: + try: + keys = get_api_keys_for_model(model, config) + except AttributeError: + keys = [] + if keys: + call_kwargs["api_key"] = keys[0] + try: + call_kwargs.update(extra_litellm_params(model, config)) + except AttributeError: + pass + call_kwargs = apply_litellm_generation_params( + call_kwargs, + model, + requested_temperature, + model_list=recovery_model_list, + ) _stream_text: Optional[str] = None _stream_usage: Dict[str, Any] = {} if stream: try: - stream_response = self._dispatch_litellm_completion( - model, - {**call_kwargs, "stream": True}, - config=config, - use_channel_router=use_channel_router, - router_model_names=router_model_names, + stream_response = call_litellm_with_param_recovery( + lambda kwargs: self._dispatch_litellm_completion( + model, + kwargs, + config=config, + use_channel_router=use_channel_router, + router_model_names=router_model_names, + ), + model=model, + call_kwargs={**call_kwargs, "stream": True}, + model_list=recovery_model_list, + cache_recovery=False, + logger=logger, ) _stream_text, _stream_usage = self._consume_litellm_stream( stream_response, @@ -2272,12 +2363,18 @@ class GeminiAnalyzer: response_validator(_stream_text) return _stream_text, model, _stream_usage - response = self._dispatch_litellm_completion( - model, - call_kwargs, - config=config, - use_channel_router=use_channel_router, - router_model_names=router_model_names, + response = call_litellm_with_param_recovery( + lambda kwargs: self._dispatch_litellm_completion( + model, + kwargs, + config=config, + use_channel_router=use_channel_router, + router_model_names=router_model_names, + ), + model=model, + call_kwargs=call_kwargs, + model_list=recovery_model_list, + logger=logger, ) content = self._extract_completion_text(response) diff --git a/src/config.py b/src/config.py index 8cd19eb0e..1ed57c2a9 100644 --- a/src/config.py +++ b/src/config.py @@ -31,6 +31,7 @@ from src.notification_noise import ( parse_notification_quiet_hours, validate_notification_timezone, ) +from src.llm import generation_params as llm_generation_params logger = logging.getLogger(__name__) @@ -61,20 +62,6 @@ _FALSEY_ENV_VALUES = {"0", "false", "no", "off"} # These are compatibility examples; actual availability should be validated by Anspire console/model entitlement. ANSPIRE_LLM_BASE_URL_DEFAULT = "https://open-gateway.anspire.cn/v6" ANSPIRE_LLM_MODEL_DEFAULT = "Doubao-Seed-2.0-lite" -# Kimi K2.6 is consumed through Moonshot's OpenAI-compatible API in this -# repository. Official references: -# - https://platform.kimi.ai/docs/guide/kimi-k2-6-quickstart -# - https://platform.moonshot.ai/docs/guide/compatibility#parameters-differences-in-request-body -# - https://huggingface.co/moonshotai/Kimi-K2.6 -# - https://docs.litellm.ai/docs/providers/openai_compatible -# Only the strict Kimi K2.6 family is normalized here; other models and -# fallbacks continue using the configured runtime temperature. -_FIXED_TEMPERATURE_LITELLM_MODELS: Dict[str, Dict[str, float]] = { - "kimi-k2.6": { - "thinking": 1.0, - "non_thinking": 0.6, - }, -} def _has_ntfy_topic_endpoint(value: Optional[str]) -> bool: @@ -350,71 +337,7 @@ def resolve_litellm_wire_model( model_list: Optional[List[Dict[str, Any]]] = None, ) -> str: """Resolve a router alias to its underlying LiteLLM wire model.""" - normalized_model = (model or "").strip() - if not normalized_model or not model_list: - return normalized_model - - model_entry = _resolve_litellm_model_list_entry(normalized_model, model_list) - if not model_entry: - return normalized_model - - params = model_entry.get("litellm_params", {}) or {} - wire_model = str(params.get("model") or "").strip() - if wire_model: - return wire_model - return normalized_model - - -def _resolve_litellm_model_list_entry( - model: str, - model_list: Optional[List[Dict[str, Any]]] = None, -) -> Optional[Dict[str, Any]]: - """Return the Router model_list entry matching the configured alias.""" - normalized_model = (model or "").strip() - if not normalized_model or not model_list: - return None - - for entry in model_list: - model_name = str(entry.get("model_name") or "").strip() - if not model_name: - params = entry.get("litellm_params", {}) or {} - model_name = str(params.get("model") or "").strip() - if model_name == normalized_model: - return entry - return None - - -def _extract_thinking_config(payload: Optional[Dict[str, Any]]) -> Any: - """Extract a thinking-mode flag from LiteLLM-style request kwargs.""" - if not isinstance(payload, dict): - return None - extra_body = payload.get("extra_body") - if isinstance(extra_body, dict) and "thinking" in extra_body: - return extra_body.get("thinking") - if "thinking" in payload: - return payload.get("thinking") - return None - - -def _parse_thinking_enabled(value: Any) -> Optional[bool]: - """Parse thinking-mode config into True/False/unknown.""" - if value is None: - return None - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized = value.strip().lower() - if normalized in {"enabled", "enable", "true", "1", "on", "thinking"}: - return True - if normalized in {"disabled", "disable", "false", "0", "off", "none", "non-thinking", "non_thinking"}: - return False - return None - if isinstance(value, dict): - if "enabled" in value: - return _parse_thinking_enabled(value.get("enabled")) - if "type" in value: - return _parse_thinking_enabled(value.get("type")) - return None + return llm_generation_params.resolve_litellm_wire_model(model, model_list) def resolve_litellm_thinking_enabled( @@ -423,19 +346,11 @@ def resolve_litellm_thinking_enabled( request_overrides: Optional[Dict[str, Any]] = None, ) -> Optional[bool]: """Resolve whether the outgoing LiteLLM request explicitly enables thinking.""" - thinking_config = None - model_entry = _resolve_litellm_model_list_entry(model, model_list) - if model_entry: - thinking_config = _extract_thinking_config(model_entry) - entry_params = model_entry.get("litellm_params", {}) or {} - entry_thinking_config = _extract_thinking_config(entry_params) - if entry_thinking_config is not None: - thinking_config = entry_thinking_config - - override_thinking_config = _extract_thinking_config(request_overrides) - if override_thinking_config is not None: - thinking_config = override_thinking_config - return _parse_thinking_enabled(thinking_config) + return llm_generation_params.resolve_litellm_thinking_enabled( + model, + model_list=model_list, + request_overrides=request_overrides, + ) def get_fixed_litellm_temperature( @@ -444,24 +359,11 @@ def get_fixed_litellm_temperature( request_overrides: Optional[Dict[str, Any]] = None, ) -> Optional[float]: """Return a provider-mandated temperature for known strict models.""" - normalized_model = resolve_litellm_wire_model(model, model_list).lower() - if not normalized_model: - return None - thinking_enabled = resolve_litellm_thinking_enabled( + return llm_generation_params.get_fixed_litellm_temperature( model, model_list=model_list, request_overrides=request_overrides, ) - model_parts = [part for part in re.split(r"[/:\s]+", normalized_model) if part] - for model_name, temperatures in _FIXED_TEMPERATURE_LITELLM_MODELS.items(): - if any(part == model_name or part.startswith(f"{model_name}-") for part in model_parts): - if thinking_enabled is False and temperatures.get("non_thinking") is not None: - return temperatures["non_thinking"] - if temperatures.get("thinking") is not None: - return temperatures["thinking"] - if temperatures.get("non_thinking") is not None: - return temperatures["non_thinking"] - return None def normalize_litellm_temperature( @@ -473,16 +375,13 @@ def normalize_litellm_temperature( request_overrides: Optional[Dict[str, Any]] = None, ) -> float: """Normalize temperature before sending a LiteLLM request.""" - fixed_temperature = get_fixed_litellm_temperature( + return llm_generation_params.normalize_litellm_temperature( model, + temperature, + default=default, model_list=model_list, request_overrides=request_overrides, ) - if fixed_temperature is not None: - return fixed_temperature - if temperature is None: - return default - return float(temperature) def resolve_unified_llm_temperature(model: str) -> float: diff --git a/src/llm/__init__.py b/src/llm/__init__.py new file mode 100644 index 000000000..1fb632256 --- /dev/null +++ b/src/llm/__init__.py @@ -0,0 +1,2 @@ +"""LLM runtime helpers.""" + diff --git a/src/llm/errors.py b/src/llm/errors.py new file mode 100644 index 000000000..e7fb72f2a --- /dev/null +++ b/src/llm/errors.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +"""LiteLLM error classification and one-shot parameter recovery.""" + +from __future__ import annotations + +import re +from typing import Any, Callable, Dict, List, Optional + +from src.llm.generation_params import ( + GenerationParamRecovery, + apply_litellm_param_recovery, + remember_litellm_generation_param_recovery, +) + +_UNSUPPORTED_PARAM_MARKERS = ( + "unsupported", + "not supported", + "unrecognized", + "unknown parameter", + "not allowed", + "invalid parameter", + "does not support", +) + +_TEMPERATURE_VALUE_PATTERN = r"-?\d+(?:\.\d+)?" +_ALLOWED_TEMPERATURE_PATTERNS = ( + re.compile( + rf"\bonly\s+(?:the\s+)?(?:default\s+)?(?:temperature\s+)?(?:value\s+)?[\(`'\"]*(?P{_TEMPERATURE_VALUE_PATTERN})(?!\w)" + ), + re.compile( + rf"\bdefault(?:\s+temperature)?(?:\s+value)?\s*(?:is|=|:)\s*[\(`'\"]*(?P{_TEMPERATURE_VALUE_PATTERN})(?!\w)" + ), +) + + +def _collect_error_text(value: Any, seen: Optional[set] = None) -> List[str]: + if seen is None: + seen = set() + if value is None: + return [] + value_id = id(value) + if value_id in seen: + return [] + seen.add(value_id) + + chunks = [str(value)] + if isinstance(value, BaseException): + chunks.extend(_collect_error_text(getattr(value, "args", None), seen)) + if isinstance(value, dict): + for item in value.values(): + chunks.extend(_collect_error_text(item, seen)) + elif isinstance(value, (list, tuple, set)): + for item in value: + chunks.extend(_collect_error_text(item, seen)) + else: + for attr in ("message", "body", "response", "llm_provider", "param"): + if hasattr(value, attr): + chunks.extend(_collect_error_text(getattr(value, attr), seen)) + return chunks + + +def _normalized_error_text(error: BaseException) -> str: + return " ".join(chunk for chunk in _collect_error_text(error) if chunk).lower() + + +def _parse_allowed_temperature(text: str) -> Optional[float]: + for segment in re.split(r"(? Optional[GenerationParamRecovery]: + """Classify explicit provider parameter errors into a safe one-shot recovery.""" + text = _normalized_error_text(error) + if not text: + return None + + if "temperature" in text: + allowed_temperature = _parse_allowed_temperature(text) + if allowed_temperature is not None: + return GenerationParamRecovery( + set_params={"temperature": allowed_temperature}, + reason="temperature_default_only", + ) + if "only" in text and "default" in text: + return GenerationParamRecovery( + omit_params=("temperature",), + reason="temperature_default_only", + ) + if any(marker in text for marker in _UNSUPPORTED_PARAM_MARKERS): + return GenerationParamRecovery( + omit_params=("temperature",), + reason="temperature_unsupported", + ) + + for param in ("top_p", "presence_penalty", "frequency_penalty", "seed"): + if param in text and any(marker in text for marker in _UNSUPPORTED_PARAM_MARKERS): + return GenerationParamRecovery( + omit_params=(param,), + reason=f"{param}_unsupported", + ) + return None + + +def call_litellm_with_param_recovery( + call: Callable[[Dict[str, Any]], Any], + *, + model: str, + call_kwargs: Dict[str, Any], + model_list: Optional[List[Dict[str, Any]]] = None, + cache_recovery: bool = True, + logger: Optional[Any] = None, + log_label: str = "[LiteLLM]", +) -> Any: + """Call LiteLLM once, then retry once for explicit generation-parameter errors.""" + effective_kwargs = dict(call_kwargs) + try: + return call(effective_kwargs) + except Exception as exc: + recovery = classify_litellm_generation_param_error(exc) + if recovery is None: + raise + retry_kwargs = apply_litellm_param_recovery(effective_kwargs, recovery) + if retry_kwargs == effective_kwargs: + raise + if logger is not None: + logger.warning( + "%s %s generation parameter rejected (%s), retrying once with request-scoped recovery", + log_label, + model, + recovery.reason, + ) + response = call(retry_kwargs) + if cache_recovery: + remember_litellm_generation_param_recovery( + model, + recovery, + model_list=model_list, + request_overrides=retry_kwargs, + ) + return response diff --git a/src/llm/generation_params.py b/src/llm/generation_params.py new file mode 100644 index 000000000..c5bd9e2d3 --- /dev/null +++ b/src/llm/generation_params.py @@ -0,0 +1,439 @@ +# -*- coding: utf-8 -*- +"""LiteLLM generation-parameter compatibility helpers.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Optional, Tuple + + +# Kimi K2.6 is consumed through Moonshot's OpenAI-compatible API in this +# repository. Official references: +# - https://platform.kimi.ai/docs/guide/kimi-k2-6-quickstart +# - https://platform.moonshot.ai/docs/guide/compatibility#parameters-differences-in-request-body +# - https://huggingface.co/moonshotai/Kimi-K2.6 +# - https://docs.litellm.ai/docs/providers/openai_compatible +_FIXED_TEMPERATURE_LITELLM_MODELS: Dict[str, Dict[str, float]] = { + "kimi-k2.6": { + "thinking": 1.0, + "non_thinking": 0.6, + }, +} + + +@dataclass(frozen=True) +class TemperatureDirective: + """Request-scoped temperature strategy for one LiteLLM model call.""" + + temperature: Optional[float] = None + omit_temperature: bool = False + reason: str = "" + + +@dataclass(frozen=True) +class GenerationParamRecovery: + """A learned request-parameter repair for a LiteLLM model call.""" + + omit_params: Tuple[str, ...] = () + set_params: Mapping[str, Any] = field(default_factory=dict) + reason: str = "" + + +_GENERATION_PARAM_RECOVERY_CACHE: Dict[str, GenerationParamRecovery] = {} + +_LITELLM_ENDPOINT_PARAM_KEYS = ( + "api_base", + "base_url", + "api_version", + "api_type", + "azure_endpoint", + "azure_deployment", + "deployment_id", + "custom_llm_provider", + "organization", + "region_name", + "aws_region_name", + "vertex_project", + "vertex_location", + "extra_headers", + "headers", + "default_headers", +) +_LITELLM_ROUTING_PARAM_KEYS = ("model", *_LITELLM_ENDPOINT_PARAM_KEYS) +_SECRET_CACHE_FIELD_NAMES = { + "api_key", + "authorization", + "proxy_authorization", + "cookie", + "set-cookie", + "x-api-key", + "api-key", + "openai-api-key", +} + + +def _resolve_litellm_model_list_entry( + model: str, + model_list: Optional[List[Dict[str, Any]]] = None, +) -> Optional[Dict[str, Any]]: + """Return the Router model_list entry matching the configured alias.""" + entries = _resolve_litellm_model_list_entries(model, model_list) + return entries[0] if entries else None + + +def _resolve_litellm_model_list_entries( + model: str, + model_list: Optional[List[Dict[str, Any]]] = None, +) -> List[Dict[str, Any]]: + """Return Router model_list entries matching the configured alias.""" + normalized_model = (model or "").strip() + if not normalized_model or not model_list: + return [] + + entries: List[Dict[str, Any]] = [] + for entry in model_list: + model_name = str(entry.get("model_name") or "").strip() + if not model_name: + params = entry.get("litellm_params", {}) or {} + model_name = str(params.get("model") or "").strip() + if model_name == normalized_model: + entries.append(entry) + return entries + + +def resolve_litellm_wire_model( + model: str, + model_list: Optional[List[Dict[str, Any]]] = None, +) -> str: + """Resolve a router alias to its underlying LiteLLM wire model.""" + normalized_model = (model or "").strip() + if not normalized_model or not model_list: + return normalized_model + + model_entry = _resolve_litellm_model_list_entry(normalized_model, model_list) + if not model_entry: + return normalized_model + + params = model_entry.get("litellm_params", {}) or {} + wire_model = str(params.get("model") or "").strip() + if wire_model: + return wire_model + return normalized_model + + +def _extract_thinking_config(payload: Optional[Dict[str, Any]]) -> Any: + """Extract a thinking-mode flag from LiteLLM-style request kwargs.""" + if not isinstance(payload, dict): + return None + extra_body = payload.get("extra_body") + if isinstance(extra_body, dict) and "thinking" in extra_body: + return extra_body.get("thinking") + if "thinking" in payload: + return payload.get("thinking") + return None + + +def _parse_thinking_enabled(value: Any) -> Optional[bool]: + """Parse thinking-mode config into True/False/unknown.""" + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"enabled", "enable", "true", "1", "on", "thinking"}: + return True + if normalized in {"disabled", "disable", "false", "0", "off", "none", "non-thinking", "non_thinking"}: + return False + return None + if isinstance(value, dict): + if "enabled" in value: + return _parse_thinking_enabled(value.get("enabled")) + if "type" in value: + return _parse_thinking_enabled(value.get("type")) + return None + + +def resolve_litellm_thinking_enabled( + model: str, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> Optional[bool]: + """Resolve whether the outgoing LiteLLM request explicitly enables thinking.""" + thinking_config = None + model_entry = _resolve_litellm_model_list_entry(model, model_list) + if model_entry: + thinking_config = _extract_thinking_config(model_entry) + entry_params = model_entry.get("litellm_params", {}) or {} + entry_thinking_config = _extract_thinking_config(entry_params) + if entry_thinking_config is not None: + thinking_config = entry_thinking_config + + override_thinking_config = _extract_thinking_config(request_overrides) + if override_thinking_config is not None: + thinking_config = override_thinking_config + return _parse_thinking_enabled(thinking_config) + + +def _model_parts(model: str) -> List[str]: + return [part for part in re.split(r"[/:\s]+", (model or "").lower()) if part] + + +def _matches_model_family(model: str, family: str) -> bool: + return any(part == family or part.startswith(f"{family}-") for part in _model_parts(model)) + + +def _should_omit_litellm_temperature(model: str) -> bool: + """Return whether a model family should rely on the provider default temperature.""" + return any( + part.startswith(("gpt-5", "gpt5")) + or part in {"o1", "o3", "o4"} + or part.startswith(("o1-", "o3-", "o4-")) + for part in _model_parts(model) + ) + + +def get_fixed_litellm_temperature( + model: str, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> Optional[float]: + """Return a provider-mandated temperature for known strict models.""" + normalized_model = resolve_litellm_wire_model(model, model_list).lower() + if not normalized_model: + return None + thinking_enabled = resolve_litellm_thinking_enabled( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + for model_name, temperatures in _FIXED_TEMPERATURE_LITELLM_MODELS.items(): + if _matches_model_family(normalized_model, model_name): + if thinking_enabled is False and temperatures.get("non_thinking") is not None: + return temperatures["non_thinking"] + if temperatures.get("thinking") is not None: + return temperatures["thinking"] + if temperatures.get("non_thinking") is not None: + return temperatures["non_thinking"] + return None + + +def resolve_litellm_temperature_directive( + model: str, + *, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> TemperatureDirective: + """Resolve the request-scoped temperature directive for a LiteLLM model.""" + fixed_temperature = get_fixed_litellm_temperature( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + if fixed_temperature is not None: + return TemperatureDirective( + temperature=fixed_temperature, + reason="fixed_model_temperature", + ) + + wire_model = resolve_litellm_wire_model(model, model_list) + if _should_omit_litellm_temperature(wire_model): + return TemperatureDirective( + omit_temperature=True, + reason="provider_default_temperature", + ) + return TemperatureDirective() + + +def normalize_litellm_temperature( + model: str, + temperature: Optional[float], + *, + default: float = 0.7, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> float: + """Return the legacy float temperature normalization for callers that need it.""" + fixed_temperature = get_fixed_litellm_temperature( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + if fixed_temperature is not None: + return fixed_temperature + if temperature is None: + return default + return float(temperature) + + +def _redact_recovery_cache_value(param_name: str, value: Any) -> Any: + if param_name.strip().lower() in _SECRET_CACHE_FIELD_NAMES: + return "" if value else "" + if isinstance(value, Mapping): + return { + str(key): _redact_recovery_cache_value(str(key), nested_value) + for key, nested_value in sorted(value.items(), key=lambda item: str(item[0])) + } + if isinstance(value, (list, tuple)): + return [_redact_recovery_cache_value(param_name, item) for item in value] + return value + + +def _stable_recovery_cache_json(value: Mapping[str, Any]) -> str: + redacted = { + key: _redact_recovery_cache_value(key, val) + for key, val in sorted(value.items()) + } + return json.dumps(redacted, sort_keys=True, separators=(",", ":"), default=str) + + +def _filter_litellm_routing_params(params: Mapping[str, Any]) -> Dict[str, Any]: + return { + key: params[key] + for key in _LITELLM_ROUTING_PARAM_KEYS + if key in params and params[key] not in (None, "") + } + + +def _request_endpoint_cache_scope(request_overrides: Optional[Dict[str, Any]]) -> Optional[str]: + if not isinstance(request_overrides, Mapping): + return None + routing_params = _filter_litellm_routing_params(request_overrides) + if not any(key in routing_params for key in _LITELLM_ENDPOINT_PARAM_KEYS): + return None + return _stable_recovery_cache_json(routing_params) + + +def _model_list_endpoint_cache_scope( + model: str, + model_list: Optional[List[Dict[str, Any]]], +) -> Optional[str]: + entries = _resolve_litellm_model_list_entries(model, model_list) + if not entries: + return "default" + + fingerprints = set() + for entry in entries: + params = entry.get("litellm_params", {}) or {} + if not isinstance(params, Mapping): + params = {} + routing_params = _filter_litellm_routing_params(params) + if not routing_params: + routing_params = {"model": str(entry.get("model_name") or model).strip()} + fingerprints.add(_stable_recovery_cache_json(routing_params)) + + if len(fingerprints) != 1: + return None + return next(iter(fingerprints)) + + +def _recovery_cache_key( + model: str, + *, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> Optional[str]: + wire_model = resolve_litellm_wire_model(model, model_list).strip().lower() + thinking_enabled = resolve_litellm_thinking_enabled( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + endpoint_scope = _request_endpoint_cache_scope(request_overrides) + if endpoint_scope is None: + endpoint_scope = _model_list_endpoint_cache_scope(model, model_list) + if endpoint_scope is None: + return None + return ( + f"{wire_model or (model or '').strip().lower()}" + f"|thinking={thinking_enabled}" + f"|endpoint={endpoint_scope}" + ) + + +def apply_litellm_param_recovery( + call_kwargs: Dict[str, Any], + recovery: GenerationParamRecovery, +) -> Dict[str, Any]: + """Return kwargs with a learned parameter recovery applied.""" + updated = dict(call_kwargs) + for param in recovery.omit_params: + updated.pop(param, None) + for param, value in recovery.set_params.items(): + updated[param] = value + return updated + + +def get_cached_litellm_generation_param_recovery( + model: str, + *, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> Optional[GenerationParamRecovery]: + """Return a process-local parameter recovery learned for this model call shape.""" + key = _recovery_cache_key( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + if key is None: + return None + return _GENERATION_PARAM_RECOVERY_CACHE.get(key) + + +def remember_litellm_generation_param_recovery( + model: str, + recovery: GenerationParamRecovery, + *, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> None: + """Remember a successful parameter recovery for later requests in this process.""" + key = _recovery_cache_key( + model, + model_list=model_list, + request_overrides=request_overrides, + ) + if key is None: + return + _GENERATION_PARAM_RECOVERY_CACHE[key] = recovery + + +def clear_litellm_generation_param_recovery_cache() -> None: + """Clear process-local learned parameter recoveries. Intended for tests.""" + _GENERATION_PARAM_RECOVERY_CACHE.clear() + + +def apply_litellm_generation_params( + call_kwargs: Dict[str, Any], + model: str, + temperature: Optional[float], + *, + default_temperature: float = 0.7, + model_list: Optional[List[Dict[str, Any]]] = None, + request_overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Return kwargs with model-compatible generation parameters applied.""" + updated = dict(call_kwargs) + effective_overrides = request_overrides if request_overrides is not None else updated + directive = resolve_litellm_temperature_directive( + model, + model_list=model_list, + request_overrides=effective_overrides, + ) + if directive.omit_temperature: + updated.pop("temperature", None) + elif directive.temperature is not None: + updated["temperature"] = directive.temperature + else: + updated["temperature"] = default_temperature if temperature is None else float(temperature) + cached_recovery = get_cached_litellm_generation_param_recovery( + model, + model_list=model_list, + request_overrides=updated, + ) + if cached_recovery: + updated = apply_litellm_param_recovery(updated, cached_recovery) + return updated diff --git a/src/services/system_config_service.py b/src/services/system_config_service.py index af2e6e2f5..eb993a9da 100644 --- a/src/services/system_config_service.py +++ b/src/services/system_config_service.py @@ -27,7 +27,6 @@ from src.config import ( channel_allows_empty_api_key, get_configured_llm_models, normalize_agent_litellm_model, - normalize_litellm_temperature, normalize_news_strategy_profile, normalize_llm_channel_model, parse_env_bool, @@ -43,6 +42,8 @@ from src.core.config_registry import ( get_field_definition, get_registered_field_keys, ) +from src.llm.errors import call_litellm_with_param_recovery +from src.llm.generation_params import apply_litellm_generation_params from src.notification_noise import validate_notification_timezone from src.notification_sender.gotify_sender import resolve_gotify_message_endpoint from src.notification_sender.ntfy_sender import resolve_ntfy_endpoint @@ -724,10 +725,6 @@ class SystemConfigService: call_kwargs: Dict[str, Any] = { "model": resolved_model, "messages": [{"role": "user", "content": "Reply with OK"}], - "temperature": normalize_litellm_temperature( - resolved_model, - self._get_runtime_llm_temperature(), - ), "max_tokens": 256, # Increased to allow MiniMax-M2.7 thinking process + response "timeout": max(5.0, float(timeout_seconds)), } @@ -735,6 +732,11 @@ class SystemConfigService: call_kwargs["api_key"] = selected_api_key if base_url.strip(): call_kwargs["api_base"] = base_url.strip() + call_kwargs = apply_litellm_generation_params( + call_kwargs, + resolved_model, + self._get_runtime_llm_temperature(), + ) try: import litellm @@ -746,7 +748,13 @@ class SystemConfigService: LLMToolAdapter._register_custom_model_pricing() started_at = time.perf_counter() - response = litellm.completion(**call_kwargs) + response = call_litellm_with_param_recovery( + lambda kwargs: litellm.completion(**kwargs), + model=resolved_model, + call_kwargs=call_kwargs, + logger=logger, + log_label="[LLM channel test]", + ) latency_ms = int((time.perf_counter() - started_at) * 1000) content, parse_error_code, parse_error, parse_reason = self._extract_llm_completion_content(response) if parse_error_code: @@ -1151,7 +1159,6 @@ class SystemConfigService: call_kwargs: Dict[str, Any] = { "model": resolved_model, "messages": messages, - "temperature": normalize_litellm_temperature(resolved_model, 0.0), "max_tokens": max_tokens, "timeout": min(max(5.0, timeout), 10.0), } @@ -1161,6 +1168,11 @@ class SystemConfigService: call_kwargs["api_base"] = base_url.strip() if extra: call_kwargs.update(extra) + call_kwargs = apply_litellm_generation_params( + call_kwargs, + resolved_model, + 0.0, + ) return call_kwargs @classmethod diff --git a/tests/test_agent_pipeline.py b/tests/test_agent_pipeline.py index eabbefd57..71acd5a09 100644 --- a/tests/test_agent_pipeline.py +++ b/tests/test_agent_pipeline.py @@ -1687,6 +1687,172 @@ class TestAgentConstructionChain(unittest.TestCase): self.assertEqual(result.content, "agent ok") self.assertEqual(mock_completion.call_args.kwargs["temperature"], 0.6) + @patch("src.agent.llm_adapter.Router") + def test_llm_adapter_omits_temperature_for_gpt5_family(self, _mock_router): + """Agent direct LiteLLM calls should omit temperature for strict default-temperature models.""" + mock_cfg = SimpleNamespace( + agent_litellm_model="", + litellm_model="openai/gpt5.5-ferr", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=[], + deepseek_api_keys=[], + openai_base_url=None, + ) + + from src.agent.llm_adapter import LLMToolAdapter + adapter = LLMToolAdapter(config=mock_cfg) + adapter._router = None + response = SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace( + content="agent ok", + tool_calls=[], + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + + with patch("src.agent.llm_adapter.litellm.completion", return_value=response) as mock_completion: + result = adapter._call_litellm_model( + [{"role": "user", "content": "hi"}], + [], + "openai/gpt5.5-ferr", + temperature=0.2, + ) + + self.assertEqual(result.content, "agent ok") + self.assertNotIn("temperature", mock_completion.call_args.kwargs) + + @patch("src.agent.llm_adapter.Router") + def test_llm_adapter_recovers_from_unsupported_temperature(self, _mock_router): + """Agent direct LiteLLM calls should retry once with a request-scoped parameter repair.""" + from src.llm.generation_params import clear_litellm_generation_param_recovery_cache + + clear_litellm_generation_param_recovery_cache() + mock_cfg = SimpleNamespace( + agent_litellm_model="", + litellm_model="openai/custom-temp-locked-agent", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=[], + deepseek_api_keys=[], + openai_base_url=None, + ) + + from src.agent.llm_adapter import LLMToolAdapter + adapter = LLMToolAdapter(config=mock_cfg) + adapter._router = None + response = SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace( + content="agent ok", + tool_calls=[], + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + + with patch("src.agent.llm_adapter.litellm.completion") as mock_completion: + mock_completion.side_effect = [ + RuntimeError("Unsupported parameter: temperature is not supported"), + response, + ] + result = adapter._call_litellm_model( + [{"role": "user", "content": "hi"}], + [], + "openai/custom-temp-locked-agent", + temperature=0.2, + ) + + self.assertEqual(result.content, "agent ok") + self.assertEqual(mock_completion.call_args_list[0].kwargs["temperature"], 0.2) + self.assertNotIn("temperature", mock_completion.call_args_list[1].kwargs) + + @patch("src.agent.llm_adapter.Router") + def test_llm_adapter_legacy_router_recovery_cache_is_scoped_to_endpoint(self, mock_router): + """Legacy multi-key Router recoveries should not leak across base URLs.""" + from src.llm.generation_params import clear_litellm_generation_param_recovery_cache + + clear_litellm_generation_param_recovery_cache() + response = SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace( + content="agent ok", + tool_calls=[], + ) + ) + ], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + strict_router = MagicMock() + flex_router = MagicMock() + strict_router.completion.side_effect = [ + RuntimeError("Unsupported parameter: temperature is not supported"), + response, + ] + flex_router.completion.return_value = response + mock_router.side_effect = [strict_router, flex_router] + + strict_cfg = SimpleNamespace( + agent_litellm_model="", + litellm_model="openai/shared-model", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=["sk-strict-key-1", "sk-strict-key-2"], + deepseek_api_keys=[], + openai_base_url="https://strict.example/v1", + ) + flex_cfg = SimpleNamespace( + agent_litellm_model="", + litellm_model="openai/shared-model", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=["sk-flex-key-1", "sk-flex-key-2"], + deepseek_api_keys=[], + openai_base_url="https://flex.example/v1", + ) + + from src.agent.llm_adapter import LLMToolAdapter + + strict_adapter = LLMToolAdapter(config=strict_cfg) + strict_result = strict_adapter._call_litellm_model( + [{"role": "user", "content": "hi"}], + [], + "openai/shared-model", + temperature=0.2, + ) + flex_adapter = LLMToolAdapter(config=flex_cfg) + flex_result = flex_adapter._call_litellm_model( + [{"role": "user", "content": "hi"}], + [], + "openai/shared-model", + temperature=0.2, + ) + + self.assertEqual(strict_result.content, "agent ok") + self.assertEqual(flex_result.content, "agent ok") + self.assertEqual(strict_router.completion.call_args_list[0].kwargs["temperature"], 0.2) + self.assertNotIn("temperature", strict_router.completion.call_args_list[1].kwargs) + self.assertEqual(flex_router.completion.call_args.kwargs["temperature"], 0.2) + @patch("src.agent.llm_adapter.Router") def test_llm_adapter_fallback_does_not_leak_kimi_fixed_temperature(self, _mock_router): """Non-Kimi fallbacks should keep the requested temperature after a Kimi failure.""" diff --git a/tests/test_llm_channel_config.py b/tests/test_llm_channel_config.py index 8babdbb72..2a948f5f7 100644 --- a/tests/test_llm_channel_config.py +++ b/tests/test_llm_channel_config.py @@ -14,6 +14,10 @@ from src.config import ( get_fixed_litellm_temperature, normalize_litellm_temperature, ) +from src.llm.generation_params import ( + apply_litellm_generation_params, + resolve_litellm_temperature_directive, +) class LLMChannelConfigTestCase(unittest.TestCase): @@ -479,6 +483,38 @@ class LLMChannelConfigTestCase(unittest.TestCase): 0.6, ) + def test_gpt5_family_temperature_is_omitted_at_request_build_time(self) -> None: + directive = resolve_litellm_temperature_directive("openai/gpt5.5-ferr") + self.assertTrue(directive.omit_temperature) + + call_kwargs = apply_litellm_generation_params( + {"model": "openai/gpt5.5-ferr", "messages": [], "temperature": 0.2}, + "openai/gpt5.5-ferr", + 0.2, + ) + + self.assertNotIn("temperature", call_kwargs) + self.assertAlmostEqual(normalize_litellm_temperature("openai/gpt5.5-ferr", 0.2), 0.2) + + def test_gpt5_temperature_directive_resolves_litellm_yaml_alias(self) -> None: + model_list = [ + { + "model_name": "future_router", + "litellm_params": {"model": "openai/gpt-5.5"}, + } + ] + + directive = resolve_litellm_temperature_directive("future_router", model_list=model_list) + call_kwargs = apply_litellm_generation_params( + {"model": "future_router", "messages": []}, + "future_router", + 0.2, + model_list=model_list, + ) + + self.assertTrue(directive.omit_temperature) + self.assertNotIn("temperature", call_kwargs) + @patch("src.config.setup_env") @patch.object(Config, "_parse_litellm_yaml", return_value=[]) def test_local_openai_compatible_channel_defaults_to_openai_protocol(self, _mock_parse_yaml, _mock_setup_env) -> None: diff --git a/tests/test_llm_param_recovery.py b/tests/test_llm_param_recovery.py new file mode 100644 index 000000000..4669ac1af --- /dev/null +++ b/tests/test_llm_param_recovery.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +"""Tests for LiteLLM generation-parameter recovery.""" + +from src.llm.errors import ( + call_litellm_with_param_recovery, + classify_litellm_generation_param_error, +) +from src.llm.generation_params import ( + apply_litellm_generation_params, + clear_litellm_generation_param_recovery_cache, +) + + +def test_temperature_default_only_error_sets_temperature_to_one() -> None: + recovery = classify_litellm_generation_param_error( + RuntimeError( + "Unsupported value: 'temperature' does not support 0.7 with this model. " + "Only the default (1.0) value is supported." + ) + ) + + assert recovery is not None + assert recovery.set_params == {"temperature": 1.0} + assert recovery.omit_params == () + + +def test_temperature_default_only_error_uses_named_default_value() -> None: + recovery = classify_litellm_generation_param_error( + RuntimeError( + "Unsupported value: 'temperature' does not support 1.0 with this model. " + "Only `0.6` is allowed." + ) + ) + + assert recovery is not None + assert recovery.set_params == {"temperature": 0.6} + assert recovery.omit_params == () + + +def test_temperature_default_only_error_without_named_value_omits_temperature() -> None: + recovery = classify_litellm_generation_param_error( + RuntimeError( + "Unsupported value: 'temperature' does not support 0.7 with this model. " + "Only the default value is supported." + ) + ) + + assert recovery is not None + assert recovery.set_params == {} + assert recovery.omit_params == ("temperature",) + + +def test_unsupported_temperature_error_retries_once_and_caches_recovery() -> None: + clear_litellm_generation_param_recovery_cache() + calls = [] + + def _call(kwargs): + calls.append(dict(kwargs)) + if len(calls) == 1: + raise RuntimeError("Unsupported parameter: temperature is not supported") + return "ok" + + result = call_litellm_with_param_recovery( + _call, + model="openai/custom-temp-locked", + call_kwargs={ + "model": "openai/custom-temp-locked", + "messages": [], + "temperature": 0.7, + }, + ) + future_kwargs = apply_litellm_generation_params( + {"model": "openai/custom-temp-locked", "messages": []}, + "openai/custom-temp-locked", + 0.7, + ) + + assert result == "ok" + assert calls[0]["temperature"] == 0.7 + assert "temperature" not in calls[1] + assert "temperature" not in future_kwargs + + +def test_recovery_cache_is_scoped_to_api_base() -> None: + clear_litellm_generation_param_recovery_cache() + calls = [] + + def _call(kwargs): + calls.append(dict(kwargs)) + if len(calls) == 1: + raise RuntimeError("Unsupported parameter: temperature is not supported") + return "ok" + + result = call_litellm_with_param_recovery( + _call, + model="openai/shared-model", + call_kwargs={ + "model": "openai/shared-model", + "messages": [], + "api_base": "https://strict.example/v1", + "temperature": 0.7, + }, + ) + strict_kwargs = apply_litellm_generation_params( + {"model": "openai/shared-model", "messages": [], "api_base": "https://strict.example/v1"}, + "openai/shared-model", + 0.7, + ) + flexible_kwargs = apply_litellm_generation_params( + {"model": "openai/shared-model", "messages": [], "api_base": "https://flex.example/v1"}, + "openai/shared-model", + 0.7, + ) + + assert result == "ok" + assert "temperature" not in strict_kwargs + assert flexible_kwargs["temperature"] == 0.7 + + +def test_recovery_cache_skips_ambiguous_router_endpoints() -> None: + clear_litellm_generation_param_recovery_cache() + model_list = [ + { + "model_name": "openai/shared-model", + "litellm_params": { + "model": "openai/shared-model", + "api_base": "https://strict.example/v1", + }, + }, + { + "model_name": "openai/shared-model", + "litellm_params": { + "model": "openai/shared-model", + "api_base": "https://flex.example/v1", + }, + }, + ] + calls = [] + + def _call(kwargs): + calls.append(dict(kwargs)) + if len(calls) == 1: + raise RuntimeError("Unsupported parameter: temperature is not supported") + return "ok" + + result = call_litellm_with_param_recovery( + _call, + model="openai/shared-model", + call_kwargs={"model": "openai/shared-model", "messages": [], "temperature": 0.7}, + model_list=model_list, + ) + future_kwargs = apply_litellm_generation_params( + {"model": "openai/shared-model", "messages": []}, + "openai/shared-model", + 0.7, + model_list=model_list, + ) + + assert result == "ok" + assert future_kwargs["temperature"] == 0.7 + + +def test_streaming_retry_does_not_cache_before_stream_is_consumed() -> None: + clear_litellm_generation_param_recovery_cache() + calls = [] + + def _broken_stream(): + raise RuntimeError("stream failed during iteration") + yield # pragma: no cover + + def _call(kwargs): + calls.append(dict(kwargs)) + if len(calls) == 1: + raise RuntimeError("Unsupported parameter: temperature is not supported") + return _broken_stream() + + stream = call_litellm_with_param_recovery( + _call, + model="openai/stream-model", + call_kwargs={ + "model": "openai/stream-model", + "messages": [], + "temperature": 0.7, + "stream": True, + }, + cache_recovery=False, + ) + try: + list(stream) + except RuntimeError: + pass + else: # pragma: no cover + raise AssertionError("stream should fail during iteration") + + future_kwargs = apply_litellm_generation_params( + {"model": "openai/stream-model", "messages": []}, + "openai/stream-model", + 0.7, + ) + + assert "temperature" not in calls[1] + assert future_kwargs["temperature"] == 0.7 diff --git a/tests/test_market_analyzer_generate_text.py b/tests/test_market_analyzer_generate_text.py index fa17fa76f..ecfd36909 100644 --- a/tests/test_market_analyzer_generate_text.py +++ b/tests/test_market_analyzer_generate_text.py @@ -142,6 +142,149 @@ class TestAnalyzerGenerateText: assert usage == {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} assert progress_updates == [3, 6] + def test_call_litellm_legacy_path_uses_legacy_model_list_for_param_recovery(self): + with patch("src.analyzer.get_config") as mock_cfg: + cfg = MagicMock() + cfg.litellm_model = "openai/gpt-4o-mini" + cfg.litellm_fallback_models = [] + cfg.gemini_api_keys = [] + cfg.anthropic_api_keys = [] + cfg.deepseek_api_keys = [] + cfg.openai_api_keys = ["sk-openai-legacy-a", "sk-openai-legacy-b"] + cfg.openai_base_url = None + cfg.llm_model_list = [ + { + "model_name": "__legacy_openai__", + "litellm_params": { + "model": "__legacy_openai__", + "api_key": "sk-openai-legacy-a", + "api_base": "https://legacy-a.example/v1", + "extra_headers": {"x-tenant": "legacy-a"}, + }, + }, + { + "model_name": "__legacy_openai__", + "litellm_params": { + "model": "__legacy_openai__", + "api_key": "sk-openai-legacy-b", + "api_base": "https://legacy-b.example/v1", + "extra_headers": {"x-tenant": "legacy-b"}, + }, + }, + ] + cfg.llm_temperature = 0.7 + mock_cfg.return_value = cfg + + from src.analyzer import GeminiAnalyzer + + analyzer = GeminiAnalyzer() + analyzer._config_override = cfg + + captured = {} + + def _fake_call_litellm_with_param_recovery(call, **kwargs): + captured["model_list"] = kwargs.get("model_list") + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + usage=None, + ) + + with patch("src.analyzer.call_litellm_with_param_recovery", side_effect=_fake_call_litellm_with_param_recovery): + text, _, _ = analyzer._call_litellm("回归用例", {"max_tokens": 128, "temperature": 0.7}) + + assert text == "ok" + passed_model_list = captured.get("model_list") + assert passed_model_list is not None + assert len(passed_model_list) == 2 + assert all(item["litellm_params"].get("model") == "openai/gpt-4o-mini" for item in passed_model_list) + assert [item["litellm_params"]["api_base"] for item in passed_model_list] == [ + "https://legacy-a.example/v1", + "https://legacy-b.example/v1", + ] + assert [item["litellm_params"]["extra_headers"] for item in passed_model_list] == [ + {"x-tenant": "legacy-a"}, + {"x-tenant": "legacy-b"}, + ] + + @patch("src.analyzer.Router") + def test_analyzer_legacy_router_recovery_cache_is_scoped_by_api_base(self, mock_router): + """Analyzer legacy recovery should not leak across same model different api_base.""" + from src.analyzer import call_litellm_with_param_recovery as real_call + from src.llm.generation_params import clear_litellm_generation_param_recovery_cache + + clear_litellm_generation_param_recovery_cache() + response = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="analyzer ok"))], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=2, total_tokens=3), + ) + strict_router = MagicMock() + flex_router = MagicMock() + strict_router.completion.side_effect = [ + RuntimeError("Unsupported parameter: temperature is not supported"), + response, + ] + flex_router.completion.return_value = response + mock_router.side_effect = [strict_router, flex_router] + + strict_cfg = SimpleNamespace( + litellm_model="openai/shared-model", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=["sk-strict-key-1", "sk-strict-key-2"], + deepseek_api_keys=[], + openai_base_url="https://strict.example/v1", + ) + flex_cfg = SimpleNamespace( + litellm_model="openai/shared-model", + litellm_fallback_models=[], + llm_model_list=[], + llm_temperature=0.2, + gemini_api_keys=[], + anthropic_api_keys=[], + openai_api_keys=["sk-flex-key-1", "sk-flex-key-2"], + deepseek_api_keys=[], + openai_base_url="https://flex.example/v1", + ) + + captured_model_lists = [] + + def _fake_recovery(call, **kwargs): + captured_model_lists.append(kwargs.get("model_list")) + return real_call(call, **kwargs) + + import src.analyzer as analyzer_module + from src.analyzer import GeminiAnalyzer + + with patch.object(analyzer_module, "call_litellm_with_param_recovery", side_effect=_fake_recovery): + GeminiAnalyzer(config=strict_cfg)._call_litellm( + "prompt", + {"max_tokens": 128, "temperature": 0.2}, + ) + GeminiAnalyzer(config=flex_cfg)._call_litellm( + "prompt", + {"max_tokens": 128, "temperature": 0.2}, + ) + + assert len(captured_model_lists) == 2 + strict_model_list = captured_model_lists[0] + flex_model_list = captured_model_lists[1] + assert strict_model_list is not None + assert flex_model_list is not None + assert all( + item.get("litellm_params", {}).get("api_base") == "https://strict.example/v1" + for item in strict_model_list + ) + assert all( + item.get("litellm_params", {}).get("api_base") == "https://flex.example/v1" + for item in flex_model_list + ) + assert strict_router.completion.call_args_list[0].kwargs["temperature"] == 0.2 + assert "temperature" not in strict_router.completion.call_args_list[1].kwargs + assert flex_router.completion.call_args.kwargs["temperature"] == 0.2 + def test_call_litellm_stream_falls_back_to_non_stream_before_first_chunk(self): analyzer = self._make_analyzer() analyzer._config_override = SimpleNamespace( @@ -315,6 +458,66 @@ class TestAnalyzerGenerateText: call_kwargs = mock_dispatch.call_args.args[1] assert call_kwargs["temperature"] == 0.6 + def test_call_litellm_omits_temperature_for_gpt5_family(self): + analyzer = self._make_analyzer() + analyzer._config_override = SimpleNamespace( + litellm_model="openai/gpt5.5-ferr", + litellm_fallback_models=[], + llm_model_list=[], + ) + response = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + with patch.object(analyzer, "_dispatch_litellm_completion", return_value=response) as mock_dispatch: + text, model_used, usage = analyzer._call_litellm( + "prompt", + {"max_tokens": 128, "temperature": 0.2}, + ) + + assert text == "ok" + assert model_used == "openai/gpt5.5-ferr" + assert usage == {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + call_kwargs = mock_dispatch.call_args.args[1] + assert "temperature" not in call_kwargs + + def test_call_litellm_recovers_from_temperature_default_error(self): + from src.llm.generation_params import clear_litellm_generation_param_recovery_cache + + clear_litellm_generation_param_recovery_cache() + analyzer = self._make_analyzer() + analyzer._config_override = SimpleNamespace( + litellm_model="openai/custom-default-temp", + litellm_fallback_models=[], + llm_model_list=[], + ) + response = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + calls = [] + + def _dispatch(model, call_kwargs, **_kwargs): + calls.append(dict(call_kwargs)) + if len(calls) == 1: + raise RuntimeError( + "temperature=0.2 is unsupported. Only the default (1.0) value is supported." + ) + return response + + with patch.object(analyzer, "_dispatch_litellm_completion", side_effect=_dispatch): + text, model_used, usage = analyzer._call_litellm( + "prompt", + {"max_tokens": 128, "temperature": 0.2}, + ) + + assert text == "ok" + assert model_used == "openai/custom-default-temp" + assert usage == {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + assert calls[0]["temperature"] == 0.2 + assert calls[1]["temperature"] == 1.0 + def test_call_litellm_keeps_user_temperature_for_non_kimi_fallback(self): analyzer = self._make_analyzer() analyzer._config_override = SimpleNamespace( diff --git a/tests/test_system_config_service.py b/tests/test_system_config_service.py index 1f8ca216f..5730ecd34 100644 --- a/tests/test_system_config_service.py +++ b/tests/test_system_config_service.py @@ -1457,6 +1457,44 @@ class SystemConfigServiceTestCase(unittest.TestCase): self.assertEqual(current_map["LITELLM_MODEL"], "openai/kimi-k2.6") self.assertEqual(current_map["LLM_TEMPERATURE"], "0.42") + def test_update_runtime_model_cleanup_does_not_rewrite_temperature(self) -> None: + self._rewrite_env( + "STOCK_LIST=600519,000001", + "LLM_CHANNELS=deepseek", + "LLM_DEEPSEEK_PROTOCOL=deepseek", + "LLM_DEEPSEEK_BASE_URL=https://api.deepseek.com", + "LLM_DEEPSEEK_API_KEY=sk-test-value", + "LLM_DEEPSEEK_MODELS=deepseek-chat,deepseek-v4-flash", + "LITELLM_MODEL=deepseek/deepseek-chat", + "AGENT_LITELLM_MODEL=deepseek/deepseek-v4-flash", + "LLM_TEMPERATURE=0.42", + "LITELLM_FALLBACK_MODELS=deepseek/deepseek-v4-flash,cohere/command-r-plus", + "VISION_MODEL=deepseek/deepseek-chat", + ) + + response = self.service.update( + config_version=self.manager.get_config_version(), + items=[ + {"key": "LLM_DEEPSEEK_MODELS", "value": "deepseek-v4-flash"}, + {"key": "LITELLM_MODEL", "value": ""}, + {"key": "AGENT_LITELLM_MODEL", "value": ""}, + {"key": "LITELLM_FALLBACK_MODELS", "value": "deepseek/deepseek-v4-flash"}, + {"key": "VISION_MODEL", "value": ""}, + ], + reload_now=False, + ) + + self.assertTrue(response["success"]) + current_map = self.manager.read_config_map() + self.assertEqual(current_map["LLM_TEMPERATURE"], "0.42") + self.assertEqual(current_map["LITELLM_MODEL"], "") + self.assertEqual(current_map["AGENT_LITELLM_MODEL"], "") + self.assertEqual(current_map["VISION_MODEL"], "") + self.assertEqual( + current_map["LITELLM_FALLBACK_MODELS"], + "deepseek/deepseek-v4-flash", + ) + @patch("litellm.completion") def test_test_llm_channel_does_not_persist_normalized_kimi_temperature(self, mock_completion) -> None: self._rewrite_env("LLM_TEMPERATURE=0.42") @@ -1480,6 +1518,62 @@ class SystemConfigServiceTestCase(unittest.TestCase): self.assertEqual(mock_completion.call_args.kwargs["temperature"], 1.0) self.assertEqual(self.manager.read_config_map()["LLM_TEMPERATURE"], "0.42") + @patch("litellm.completion") + def test_test_llm_channel_omits_temperature_for_gpt5_family(self, mock_completion) -> None: + mock_completion.return_value = type( + "MockResponse", + (), + { + "choices": [type("Choice", (), {"message": type("Message", (), {"content": "OK"})()})()], + }, + )() + + payload = self.service.test_llm_channel( + name="primary", + protocol="openai", + base_url="https://api.example.com/v1", + api_key="sk-test-value", + models=["gpt5.5-ferr"], + ) + + self.assertTrue(payload["success"]) + self.assertEqual(payload["resolved_model"], "openai/gpt5.5-ferr") + self.assertNotIn("temperature", mock_completion.call_args.kwargs) + + @patch("litellm.completion") + @patch("src.services.system_config_service.Config._load_from_env") + def test_test_llm_channel_recovers_from_unsupported_temperature( + self, + mock_load_config, + mock_completion, + ) -> None: + from src.llm.generation_params import clear_litellm_generation_param_recovery_cache + + clear_litellm_generation_param_recovery_cache() + mock_load_config.return_value = SimpleNamespace(llm_temperature=0.42) + mock_completion.side_effect = [ + RuntimeError("Unsupported parameter: temperature is not supported"), + type( + "MockResponse", + (), + { + "choices": [type("Choice", (), {"message": type("Message", (), {"content": "OK"})()})()], + }, + )(), + ] + + payload = self.service.test_llm_channel( + name="primary", + protocol="openai", + base_url="https://api.example.com/v1", + api_key="sk-test-value", + models=["custom-temp-locked-settings"], + ) + + self.assertTrue(payload["success"]) + self.assertEqual(mock_completion.call_args_list[0].kwargs["temperature"], 0.42) + self.assertNotIn("temperature", mock_completion.call_args_list[1].kwargs) + @patch("litellm.completion") @patch("src.services.system_config_service.Config._load_from_env") def test_test_llm_channel_uses_runtime_temperature_for_non_kimi_models(