From cdc8ab78b5f3a82b5ce5d49d1b361e7672e87d09 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 14 Jul 2026 18:39:30 +0800 Subject: [PATCH] feat(aiproxy): move visual provider to model columns and support streaming responses (#25144) Promote visual_provider_id/visual_model_key to dedicated ai_model columns, force non-stream upstream orchestration for visual paths, and synthesize SSE chunks for streaming Responses requests that include images. --- docs/aiproxy/functional-test.md | 2 +- pkg/aiproxy/codexconfig/catalog.go | 54 ++++++- .../codexconfig/catalog_visual_test.go | 47 +++++++ pkg/aiproxy/codexconfig/codexconfig.go | 7 +- pkg/aiproxy/codexconfig/codexconfig_test.go | 18 ++- pkg/aiproxy/codexconfig/routing_filter.go | 75 ++++++++-- pkg/aiproxy/extensions/visual/client.go | 11 ++ pkg/aiproxy/extensions/visual/config.go | 17 ++- pkg/aiproxy/extensions/visual/messages.go | 8 +- pkg/aiproxy/extensions/visual/responses.go | 26 +++- pkg/aiproxy/extensions/visual/visual_test.go | 38 ++++- pkg/aiproxy/handlers/responses.go | 81 ++++++++++- pkg/aiproxy/models/ai_models.go | 96 +++++++++++++ pkg/aiproxy/models/chat_upstream.go | 42 +++--- pkg/aiproxy/models/list_models.go | 40 +++++- pkg/aiproxy/models/list_models_test.go | 45 +++++- .../providers/openai/responses_compat_test.go | 55 ++++++++ .../openai/responses_stream_compat.go | 132 ++++++++++++++++++ pkg/apis/aiproxy/ai_model.go | 40 +++--- pkg/mcclient/options/aiproxy/resources.go | 80 +++++++++-- 20 files changed, 817 insertions(+), 97 deletions(-) diff --git a/docs/aiproxy/functional-test.md b/docs/aiproxy/functional-test.md index 832d92db38..bf7d27b222 100644 --- a/docs/aiproxy/functional-test.md +++ b/docs/aiproxy/functional-test.md @@ -301,7 +301,7 @@ startup_timeout_sec = 3600 tool_timeout_sec = 3600 ``` -**models_catalog.json**（节选）：包含该 virtual key 可见模型的 Codex 元数据（`base_instructions`、`truncation_policy`、`shell_type` 等），避免 Codex 回退到内置 preset。 +**models_catalog.json**（节选）：包含该 virtual key 可见模型的 Codex 元数据（`base_instructions`、`truncation_policy`、`shell_type`、`input_modalities` 等），避免 Codex 回退到内置 preset。若绑定 `ai_model` 已启用 Visual（`visual_active`：`config.extensions.visual.enabled` + `visual_provider_id` + `visual_model_key`），对应条目会写入 `input_modalities: ["text","image"]`，使 Codex 按多模态发图。 **aiproxy.env** diff --git a/pkg/aiproxy/codexconfig/catalog.go b/pkg/aiproxy/codexconfig/catalog.go index 52e7fbf988..7a486af52d 100644 --- a/pkg/aiproxy/codexconfig/catalog.go +++ b/pkg/aiproxy/codexconfig/catalog.go @@ -207,8 +207,50 @@ func newModelInfo(slug, displayName, description string, contextWindow int, inpu } } +// ensureImageModality appends "image" when missing, defaulting empty lists to ["text"]. +func ensureImageModality(modalities []string) []string { + hasImage := false + hasText := false + for _, m := range modalities { + switch strings.TrimSpace(m) { + case "image": + hasImage = true + case "text": + hasText = true + } + } + if hasImage { + return modalities + } + out := append([]string(nil), modalities...) + if !hasText && len(out) == 0 { + out = []string{"text"} + } else if !hasText { + out = append([]string{"text"}, out...) + } + return append(out, "image") +} + +// ApplyVisualModalities sets input_modalities to include image for Visual-active slugs. +func ApplyVisualModalities(catalog []ModelInfo, visualActiveSlugs map[string]struct{}) []ModelInfo { + if len(catalog) == 0 || len(visualActiveSlugs) == 0 { + return catalog + } + out := make([]ModelInfo, len(catalog)) + copy(out, catalog) + for i := range out { + if _, ok := visualActiveSlugs[out[i].Slug]; !ok { + continue + } + out[i].InputModalities = ensureImageModality(out[i].InputModalities) + out[i].SupportsImageDetailOriginal = true + } + return out +} + // EnsureModelInCatalog appends a fallback catalog entry when model is not listed. -func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo { +// When visualActiveSlugs contains the model, the fallback includes image modality. +func EnsureModelInCatalog(catalog []ModelInfo, model string, visualActiveSlugs map[string]struct{}) []ModelInfo { model = strings.TrimSpace(model) if model == "" { return catalog @@ -218,7 +260,15 @@ func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo { return catalog } } - return append(catalog, buildModelInfoFromEntry(ModelListEntry{ID: model})) + entry := ModelListEntry{ID: model} + if _, ok := visualActiveSlugs[model]; ok { + entry.InputModalities = []string{"text", "image"} + } + info := buildModelInfoFromEntry(entry) + if _, ok := visualActiveSlugs[model]; ok { + info.SupportsImageDetailOriginal = true + } + return append(catalog, info) } // CatalogContextWindow returns context_window for model when present in catalog. diff --git a/pkg/aiproxy/codexconfig/catalog_visual_test.go b/pkg/aiproxy/codexconfig/catalog_visual_test.go index 2dc7e74292..a3946aa7e4 100644 --- a/pkg/aiproxy/codexconfig/catalog_visual_test.go +++ b/pkg/aiproxy/codexconfig/catalog_visual_test.go @@ -29,3 +29,50 @@ func TestBuildCatalogVisualModalities(t *testing.T) { t.Fatalf("modalities = %#v", catalog[0].InputModalities) } } + +func TestApplyVisualModalities(t *testing.T) { + catalog := BuildCatalogFromIDs([]ModelListEntry{ + {ID: "text-only", OwnedBy: "deepseek"}, + {ID: "route/flash", OwnedBy: "deepseek", InputModalities: []string{"text"}}, + {ID: "already-image", OwnedBy: "deepseek", InputModalities: []string{"text", "image"}}, + }) + visual := map[string]struct{}{ + "route/flash": {}, + "already-image": {}, + "missing-slug": {}, + } + out := ApplyVisualModalities(catalog, visual) + if len(out) != 3 { + t.Fatalf("len = %d", len(out)) + } + bySlug := map[string]ModelInfo{} + for _, m := range out { + bySlug[m.Slug] = m + } + if len(bySlug["text-only"].InputModalities) != 1 || bySlug["text-only"].InputModalities[0] != "text" { + t.Fatalf("text-only = %#v", bySlug["text-only"].InputModalities) + } + if !bySlug["route/flash"].SupportsImageDetailOriginal { + t.Fatal("route/flash should support image detail original") + } + mods := bySlug["route/flash"].InputModalities + if len(mods) != 2 || mods[0] != "text" || mods[1] != "image" { + t.Fatalf("route/flash modalities = %#v", mods) + } + mods = bySlug["already-image"].InputModalities + if len(mods) != 2 || mods[1] != "image" { + t.Fatalf("already-image modalities = %#v", mods) + } +} + +func TestEnsureImageModality(t *testing.T) { + if got := ensureImageModality(nil); len(got) != 2 || got[0] != "text" || got[1] != "image" { + t.Fatalf("nil = %#v", got) + } + if got := ensureImageModality([]string{"text", "image"}); len(got) != 2 { + t.Fatalf("already has image = %#v", got) + } + if got := ensureImageModality([]string{"audio"}); len(got) != 3 || got[0] != "text" || got[2] != "image" { + t.Fatalf("preserve audio = %#v", got) + } +} diff --git a/pkg/aiproxy/codexconfig/codexconfig.go b/pkg/aiproxy/codexconfig/codexconfig.go index 4d50b7caf5..e2173a36b1 100644 --- a/pkg/aiproxy/codexconfig/codexconfig.go +++ b/pkg/aiproxy/codexconfig/codexconfig.go @@ -180,7 +180,12 @@ func Run(session *mcclient.ClientSession, opts *Options) error { if err != nil { return err } - catalog := EnsureModelInCatalog(BuildCatalogFromIDs(entries), model) + visualSlugs, err := resolveVisualActiveClientModelIDs(session, opts.Routing) + if err != nil { + return err + } + catalog := ApplyVisualModalities(BuildCatalogFromIDs(entries), visualSlugs) + catalog = EnsureModelInCatalog(catalog, model, visualSlugs) providerName := strings.TrimSpace(opts.ProviderName) if providerName == "" { diff --git a/pkg/aiproxy/codexconfig/codexconfig_test.go b/pkg/aiproxy/codexconfig/codexconfig_test.go index 9fcf9c8c80..32a6911457 100644 --- a/pkg/aiproxy/codexconfig/codexconfig_test.go +++ b/pkg/aiproxy/codexconfig/codexconfig_test.go @@ -157,14 +157,28 @@ func TestBuildCatalogFromIDs(t *testing.T) { } func TestEnsureModelInCatalog(t *testing.T) { - catalog := EnsureModelInCatalog(nil, "custom-model") + catalog := EnsureModelInCatalog(nil, "custom-model", nil) if len(catalog) != 1 || catalog[0].Slug != "custom-model" { t.Fatalf("catalog = %+v", catalog) } - catalog = EnsureModelInCatalog(catalog, "custom-model") + if len(catalog[0].InputModalities) != 1 || catalog[0].InputModalities[0] != "text" { + t.Fatalf("default modalities = %#v", catalog[0].InputModalities) + } + catalog = EnsureModelInCatalog(catalog, "custom-model", nil) if len(catalog) != 1 { t.Fatalf("duplicate append: %+v", catalog) } + visual := map[string]struct{}{"visual-model": {}} + catalog = EnsureModelInCatalog(nil, "visual-model", visual) + if len(catalog) != 1 || catalog[0].Slug != "visual-model" { + t.Fatalf("visual catalog = %+v", catalog) + } + if len(catalog[0].InputModalities) != 2 || catalog[0].InputModalities[1] != "image" { + t.Fatalf("visual modalities = %#v", catalog[0].InputModalities) + } + if !catalog[0].SupportsImageDetailOriginal { + t.Fatal("expected SupportsImageDetailOriginal") + } } func TestWriteModelsCatalogJSON(t *testing.T) { diff --git a/pkg/aiproxy/codexconfig/routing_filter.go b/pkg/aiproxy/codexconfig/routing_filter.go index 8799151f03..5862dd96d9 100644 --- a/pkg/aiproxy/codexconfig/routing_filter.go +++ b/pkg/aiproxy/codexconfig/routing_filter.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/pkg/errors" apmodels "yunion.io/x/onecloud/pkg/aiproxy/models" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" "yunion.io/x/onecloud/pkg/mcclient" apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" ) @@ -38,9 +39,41 @@ func filterModelEntriesForRouting(session *mcclient.ClientSession, routingNameOr } func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) { + routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID) + if err != nil { + return nil, err + } + ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers) + allowed := make(map[string]struct{}, len(ids)) + for _, id := range ids { + allowed[id] = struct{}{} + } + return allowed, nil +} + +// resolveVisualActiveClientModelIDs returns client model ids with VisualActive() when --routing is set. +func resolveVisualActiveClientModelIDs(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) { + routingNameOrID = strings.TrimSpace(routingNameOrID) + if routingNameOrID == "" { + return nil, nil + } + routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID) + if err != nil { + return nil, err + } + return apmodels.VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers), nil +} + +func loadRoutingCatalogContext(session *mcclient.ClientSession, routingNameOrID string) ( + *apmodels.SAiRouting, + []apmodels.SAiRoutingModel, + map[string]*apmodels.SAiModel, + map[string]*apmodels.SAiProvider, + error, +) { routingObj, err := apmodules.AiRoutings.Get(session, routingNameOrID, nil) if err != nil { - return nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID) + return nil, nil, nil, nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID) } routingID, _ := routingObj.GetString("id") routing := &apmodels.SAiRouting{ @@ -50,23 +83,17 @@ func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNam bindings, providerIDs, err := fetchRoutingBindings(session, routingID) if err != nil { - return nil, err + return nil, nil, nil, nil, err } modelsById, err := fetchCatalogModelsForRouting(session, routingID) if err != nil { - return nil, err + return nil, nil, nil, nil, err } providers, err := fetchProvidersByIDs(session, providerIDs) if err != nil { - return nil, err + return nil, nil, nil, nil, err } - - ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers) - allowed := make(map[string]struct{}, len(ids)) - for _, id := range ids { - allowed[id] = struct{}{} - } - return allowed, nil + return routing, bindings, modelsById, providers, nil } func fetchRoutingBindings(session *mcclient.ClientSession, routingID string) ([]apmodels.SAiRoutingModel, []string, error) { @@ -100,6 +127,7 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str params := jsonutils.NewDict() params.Set("ai_routing_id", jsonutils.NewString(routingID)) params.Set("enabled", jsonutils.JSONTrue) + params.Set("details", jsonutils.JSONTrue) params.Set("limit", jsonutils.NewInt(500)) result, err := apmodules.AiModels.List(session, params) if err != nil { @@ -111,14 +139,33 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str if id == "" { continue } - out[id] = &apmodels.SAiModel{ - AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")), - ModelKey: strings.TrimSpace(mustString(item, "model_key")), + mdl := &apmodels.SAiModel{ + AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")), + ModelKey: strings.TrimSpace(mustString(item, "model_key")), + VisualProviderId: strings.TrimSpace(mustString(item, "visual_provider_id")), + VisualModelKey: strings.TrimSpace(mustString(item, "visual_model_key")), + Config: parseAiModelConfig(item), } + out[id] = mdl } return out, nil } +func parseAiModelConfig(obj jsonutils.JSONObject) *api.SAiModelConfig { + if obj == nil { + return nil + } + cfgObj, err := obj.Get("config") + if err != nil || cfgObj == nil || cfgObj == jsonutils.JSONNull { + return nil + } + cfg := &api.SAiModelConfig{} + if err := cfgObj.Unmarshal(cfg); err != nil { + return nil + } + return cfg +} + func fetchProvidersByIDs(session *mcclient.ClientSession, ids []string) (map[string]*apmodels.SAiProvider, error) { out := make(map[string]*apmodels.SAiProvider, len(ids)) for _, id := range ids { diff --git a/pkg/aiproxy/extensions/visual/client.go b/pkg/aiproxy/extensions/visual/client.go index 33409b3248..f527fd2f87 100644 --- a/pkg/aiproxy/extensions/visual/client.go +++ b/pkg/aiproxy/extensions/visual/client.go @@ -83,6 +83,7 @@ func callChatCompletions(ctx context.Context, up *models.ChatUpstream, body *jso if up == nil || body == nil { return nil, fmt.Errorf("nil upstream or body") } + forceNonStreamChatBody(body) req := &upstream.Request{ URL: openai.ChatCompletionsURL(ChatBaseURL(up.BaseURL)), APIKey: up.APIKey, @@ -111,6 +112,16 @@ func ChatBaseURL(baseURL string) string { return base } +// forceNonStreamChatBody ensures visual orchestration always calls upstream chat/completions +// without stream=true (Codex Responses requests carry stream=true in the converted body). +func forceNonStreamChatBody(body *jsonutils.JSONDict) { + if body == nil { + return + } + body.Remove("stream") + body.Remove("stream_options") +} + func finishReasonFromChatResponse(body []byte) string { var resp struct { Choices []struct { diff --git a/pkg/aiproxy/extensions/visual/config.go b/pkg/aiproxy/extensions/visual/config.go index b350909ebd..6a7b0a976d 100644 --- a/pkg/aiproxy/extensions/visual/config.go +++ b/pkg/aiproxy/extensions/visual/config.go @@ -14,7 +14,12 @@ package visual -import api "yunion.io/x/onecloud/pkg/apis/aiproxy" +import ( + "strings" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) // RuntimeConfig is the resolved visual extension settings for one request. type RuntimeConfig struct { @@ -40,7 +45,11 @@ func RuntimeConfigFromModel(cfg *api.SAiModelConfig) (RuntimeConfig, *api.SAiMod return out, vis } -// Enabled reports whether visual extension is enabled on the model config. -func Enabled(cfg *api.SAiModelConfig) bool { - return cfg != nil && cfg.VisualEnabled() +// Enabled reports whether visual extension is active on the resolved upstream +// (config.enabled plus visual_provider_id / visual_model_key columns). +func Enabled(up *models.ChatUpstream) bool { + if up == nil || !up.ModelConfig.VisualEnabled() { + return false + } + return strings.TrimSpace(up.VisualProviderId) != "" && strings.TrimSpace(up.VisualModelKey) != "" } diff --git a/pkg/aiproxy/extensions/visual/messages.go b/pkg/aiproxy/extensions/visual/messages.go index 30c29ec516..2793a1df08 100644 --- a/pkg/aiproxy/extensions/visual/messages.go +++ b/pkg/aiproxy/extensions/visual/messages.go @@ -73,7 +73,7 @@ func anthropicContentHasImage(raw json.RawMessage) bool { // ShouldHandleMessages reports whether the Messages visual path should run (non-stream only). func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { - if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + if isStream || dict == nil || up == nil || !Enabled(up) { return false } return AnthropicMessagesHasImage(dict) @@ -81,7 +81,7 @@ func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isS // ShouldRejectMessagesStreaming reports stream+visual+image (unsupported). func ShouldRejectMessagesStreaming(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { - if !isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + if !isStream || dict == nil || up == nil || !Enabled(up) { return false } return AnthropicMessagesHasImage(dict) @@ -94,7 +94,7 @@ func HandleMessagesCreate( textUp *models.ChatUpstream, ) ([]byte, error) { runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig) - if visCfg == nil { + if visCfg == nil || !visCfg.Enabled { return nil, fmt.Errorf("visual config is missing") } userCred := auth.AdminCredential() @@ -102,7 +102,7 @@ func HandleMessagesCreate( if err != nil { return nil, err } - visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg) + visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey) if err != nil { return nil, err } diff --git a/pkg/aiproxy/extensions/visual/responses.go b/pkg/aiproxy/extensions/visual/responses.go index 46662f997c..cf9c02fb20 100644 --- a/pkg/aiproxy/extensions/visual/responses.go +++ b/pkg/aiproxy/extensions/visual/responses.go @@ -26,21 +26,23 @@ import ( ) // ShouldHandle reports whether the Responses visual orchestration path should run. +// Streaming requests with images use non-streaming orchestration and synthetic SSE (moon-bridge pattern). func ShouldHandle(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { - if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + _ = isStream + if dict == nil || up == nil || !Enabled(up) { return false } return openai.ResponsesInputHasImage(dict) } -// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request. -func HandleResponsesCreate( +// HandleResponsesCreateChat runs visual orchestration and returns the upstream chat completion body. +func HandleResponsesCreateChat( ctx context.Context, dict *jsonutils.JSONDict, textUp *models.ChatUpstream, ) ([]byte, *openai.ResponsesConvertState, error) { runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig) - if visCfg == nil { + if visCfg == nil || !visCfg.Enabled { return nil, nil, fmt.Errorf("visual config is missing") } userCred := auth.AdminCredential() @@ -48,7 +50,7 @@ func HandleResponsesCreate( if err != nil { return nil, nil, err } - visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg) + visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey) if err != nil { return nil, nil, err } @@ -57,6 +59,7 @@ func HandleResponsesCreate( return nil, nil, err } chatClone := cloneDict(chatBody) + forceNonStreamChatBody(chatClone) textUpChat := *textUp textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL) visUpChat := *visUp @@ -65,6 +68,19 @@ func HandleResponsesCreate( if err != nil { return nil, nil, err } + return respBody, state, nil +} + +// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request. +func HandleResponsesCreate( + ctx context.Context, + dict *jsonutils.JSONDict, + textUp *models.ChatUpstream, +) ([]byte, *openai.ResponsesConvertState, error) { + respBody, state, err := HandleResponsesCreateChat(ctx, dict, textUp) + if err != nil { + return nil, nil, err + } out, err := openai.ChatCompletionToResponses(respBody, state) if err != nil { return nil, nil, err diff --git a/pkg/aiproxy/extensions/visual/visual_test.go b/pkg/aiproxy/extensions/visual/visual_test.go index 3825fbd0e0..caba0bc353 100644 --- a/pkg/aiproxy/extensions/visual/visual_test.go +++ b/pkg/aiproxy/extensions/visual/visual_test.go @@ -66,9 +66,18 @@ func TestEnabled(t *testing.T) { Visual: &api.SAiModelVisualConfig{Enabled: true}, }, } - if !Enabled(cfg) { + up := &models.ChatUpstream{ + ModelConfig: cfg, + VisualProviderId: "prov-1", + VisualModelKey: "vision-model", + } + if !Enabled(up) { t.Fatal("expected enabled") } + up.VisualProviderId = "" + if Enabled(up) { + t.Fatal("expected disabled without visual_provider_id") + } } func TestInjectChatTools(t *testing.T) { @@ -119,6 +128,19 @@ func TestAnthropicMessagesHasImage(t *testing.T) { } } +func TestForceNonStreamChatBody(t *testing.T) { + body := jsonutils.NewDict() + body.Set("stream", jsonutils.JSONTrue) + body.Set("stream_options", jsonutils.NewDict()) + forceNonStreamChatBody(body) + if stream, _ := body.Bool("stream"); stream { + t.Fatal("expected stream removed") + } + if _, err := body.Get("stream_options"); err == nil { + t.Fatal("expected stream_options removed") + } +} + func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) { cfg := &api.SAiModelConfig{ Extensions: &api.SAiModelExtensions{ @@ -126,14 +148,20 @@ func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) { }, } up := &models.ChatUpstream{ - ModelConfig: cfg, - APIMode: "anthropic", + ModelConfig: cfg, + VisualProviderId: "prov-1", + VisualModelKey: "vision-model", + APIMode: "anthropic", } body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`)) if !ShouldHandle(body.(*jsonutils.JSONDict), up, false) { t.Fatal("expected visual handle even with anthropic api_mode") } - if ShouldHandle(body.(*jsonutils.JSONDict), up, true) { - t.Fatal("stream should not use visual orchestrator") + if !ShouldHandle(body.(*jsonutils.JSONDict), up, true) { + t.Fatal("stream with image should use visual orchestrator") + } + bodyText, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}]}`)) + if ShouldHandle(bodyText.(*jsonutils.JSONDict), up, true) { + t.Fatal("stream without image should not use visual orchestrator") } } diff --git a/pkg/aiproxy/handlers/responses.go b/pkg/aiproxy/handlers/responses.go index 39ef5eb975..8ee925365c 100644 --- a/pkg/aiproxy/handlers/responses.go +++ b/pkg/aiproxy/handlers/responses.go @@ -106,19 +106,42 @@ func handleResponsesCreate(ctx context.Context, w http.ResponseWriter, r *http.R } if visual.ShouldHandle(dict, up, isStream) { - bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up) + if !isStream { + bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up) + if err != nil { + dbg.Error("visual orchestration: %v", err) + writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err) + return + } + dbg.ClientResponse(bodyOut) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bodyOut) + if up.AiKeyId != "" { + models.RecordAiKeySuccess(up.AiKeyId) + } + return + } + + chatBody, _, err := visual.HandleResponsesCreateChat(ctx, dict, up) if err != nil { dbg.Error("visual orchestration: %v", err) writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err) return } - dbg.ClientResponse(bodyOut) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write(bodyOut) - if up.AiKeyId != "" { - models.RecordAiKeySuccess(up.AiKeyId) + payloads, err := openai.NonStreamChatCompletionToStreamPayloads(chatBody) + if err != nil { + dbg.Error("visual synthetic stream: %v", err) + writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual synthetic stream: %v", err) + return } + adapter, err := responses.GetAdapter(up.ProviderKey, up.APIMode) + if err != nil { + dbg.Error("adapter: %v", err) + writeResponsesError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err) + return + } + writeResponsesSyntheticStream(ctx, w, adapter, up.UpstreamModel, dict, payloads, up.AiKeyId, dbg) return } @@ -428,6 +451,50 @@ func writeResponsesPassthroughStream(ctx context.Context, w http.ResponseWriter, } } +func writeResponsesSyntheticStream( + ctx context.Context, + w http.ResponseWriter, + adapter providerapi.ResponsesAdapter, + requestModel string, + requestBody *jsonutils.JSONDict, + payloads [][]byte, + aiKeyId string, + dbg *ProxyDebugSession, +) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flushIf(w) + + state := adapter.NewStreamState(requestModel, requestBody) + streamOK := true + outSeq := 0 + for _, payload := range payloads { + events, err := adapter.ConvertStreamPayload(state, payload, false) + if err != nil { + dbg.Error("visual synthetic stream convert error: %v", err) + streamOK = false + break + } + outSeq++ + writeResponsesSSEEvents(w, events, dbg, outSeq) + } + if streamOK { + events, err := adapter.ConvertStreamPayload(state, nil, true) + if err != nil { + dbg.Error("visual synthetic stream end error: %v", err) + streamOK = false + } else { + outSeq++ + writeResponsesSSEEvents(w, events, dbg, outSeq) + } + } + if streamOK && aiKeyId != "" { + models.RecordAiKeySuccess(aiKeyId) + } +} + func writeResponsesTranslatedStream( ctx context.Context, w http.ResponseWriter, diff --git a/pkg/aiproxy/models/ai_models.go b/pkg/aiproxy/models/ai_models.go index 7fa3c96e0a..f35d9f7ec3 100644 --- a/pkg/aiproxy/models/ai_models.go +++ b/pkg/aiproxy/models/ai_models.go @@ -25,6 +25,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/aiproxy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -36,10 +37,22 @@ type SAiModel struct { AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` // ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo). ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"` + // VisualProviderId is the ai_provider used for tool-delegated image analysis. + VisualProviderId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"` + // VisualModelKey is the upstream model id for visual analysis. + VisualModelKey string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` // Config stores per-model extension settings (e.g. visual delegation). Config *api.SAiModelConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"` } +// VisualActive reports whether visual extension is enabled and columns are set. +func (m *SAiModel) VisualActive() bool { + if m == nil || !m.Config.VisualEnabled() { + return false + } + return strings.TrimSpace(m.VisualProviderId) != "" && strings.TrimSpace(m.VisualModelKey) != "" +} + type SAiModelManager struct { db.SEnabledStatusStandaloneResourceBaseManager } @@ -128,20 +141,43 @@ func (manager *SAiModelManager) FetchCustomizeColumns( rows := make([]api.AiModelDetails, len(objs)) baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) providerIds := make([]string, len(objs)) + visualProviderIds := make([]string, 0, len(objs)) for i := range objs { rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i] m := objs[i].(*SAiModel) providerIds[i] = m.AiProviderId + if vid := strings.TrimSpace(m.VisualProviderId); vid != "" { + visualProviderIds = append(visualProviderIds, vid) + } } providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds) if err != nil { log.Errorf("FetchIdNameMap2 ai_provider: %v", err) return rows } + visualProviderNames, err := db.FetchIdNameMap2(AiProviderManager, visualProviderIds) + if err != nil { + log.Errorf("FetchIdNameMap2 visual ai_provider: %v", err) + visualProviderNames = nil + } + visualProviderKeys, err := db.FetchIdFieldMap2(AiProviderManager, "provider_key", visualProviderIds) + if err != nil { + log.Errorf("FetchIdFieldMap2 visual ai_provider provider_key: %v", err) + visualProviderKeys = nil + } for i := range rows { rows[i].AiProviderName, _ = providerNames[providerIds[i]] m := objs[i].(*SAiModel) + rows[i].VisualProviderId = m.VisualProviderId + rows[i].VisualModelKey = m.VisualModelKey + rows[i].VisualActive = m.VisualActive() rows[i].Config = m.Config + if visualProviderNames != nil { + rows[i].VisualProviderName, _ = visualProviderNames[m.VisualProviderId] + } + if visualProviderKeys != nil { + rows[i].VisualProviderKey, _ = visualProviderKeys[m.VisualProviderId] + } } return rows } @@ -179,6 +215,16 @@ func (manager *SAiModelManager) ValidateCreateData( input.Name = defaultAiModelName(prov.Name, mk) } + vpid, vmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey) + if err != nil { + return input, err + } + input.VisualProviderId = vpid + input.VisualModelKey = vmk + if err := validateAiModelVisualSettings(input.Config, vpid, vmk); err != nil { + return input, err + } + return input, nil } @@ -219,5 +265,55 @@ func (m *SAiModel) ValidateUpdateData( } } + cfg := m.Config + if input.Config != nil { + cfg = input.Config + } + vpid := m.VisualProviderId + vmk := m.VisualModelKey + if strings.TrimSpace(input.VisualProviderId) != "" || strings.TrimSpace(input.VisualModelKey) != "" { + nvpid, nvmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey) + if err != nil { + return input, err + } + if strings.TrimSpace(input.VisualProviderId) != "" { + vpid = nvpid + input.VisualProviderId = nvpid + } + if strings.TrimSpace(input.VisualModelKey) != "" { + vmk = nvmk + input.VisualModelKey = nvmk + } + } + if err := validateAiModelVisualSettings(cfg, vpid, vmk); err != nil { + return input, err + } + return input, nil } + +func normalizeAiModelVisualFields( + ctx context.Context, + userCred mcclient.TokenCredential, + visualProviderId, visualModelKey string, +) (string, string, error) { + vpid := strings.TrimSpace(visualProviderId) + vmk := strings.TrimSpace(visualModelKey) + if vpid != "" { + prov, err := fetchEnabledAiProvider(ctx, userCred, vpid) + if err != nil { + return "", "", errors.Wrap(err, "visual_provider_id") + } + vpid = prov.Id + } + return vpid, vmk, nil +} + +func validateAiModelVisualSettings(cfg *api.SAiModelConfig, visualProviderId, visualModelKey string) error { + vpid := strings.TrimSpace(visualProviderId) + vmk := strings.TrimSpace(visualModelKey) + if cfg.VisualEnabled() && (vpid == "" || vmk == "") { + return errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required when visual is enabled") + } + return nil +} diff --git a/pkg/aiproxy/models/chat_upstream.go b/pkg/aiproxy/models/chat_upstream.go index f59b7b1905..649dd408d3 100644 --- a/pkg/aiproxy/models/chat_upstream.go +++ b/pkg/aiproxy/models/chat_upstream.go @@ -72,6 +72,9 @@ type ChatUpstream struct { AiKeyId string APIMode string ModelConfig *api.SAiModelConfig + // VisualProviderId / VisualModelKey come from the resolved text ai_model row. + VisualProviderId string + VisualModelKey string // VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row. VirtualKeyId string @@ -346,19 +349,21 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, } up := &ChatUpstream{ - BaseURL: baseURL, - APIKey: keyRes.Secret, - UpstreamModel: upstreamModel, - ProviderKey: prov.ProviderKey, - AiProviderId: prov.Id, - AiModelId: mdl.Id, - AiKeyId: keyRes.AiKeyId, - VirtualKeyId: vk.Id, - APIMode: apiMode, - ModelConfig: mdl.Config, - ProjectId: vk.ProjectId, - DomainId: vk.DomainId, - RoutingLog: resolved.routingLog, + BaseURL: baseURL, + APIKey: keyRes.Secret, + UpstreamModel: upstreamModel, + ProviderKey: prov.ProviderKey, + AiProviderId: prov.Id, + AiModelId: mdl.Id, + AiKeyId: keyRes.AiKeyId, + VirtualKeyId: vk.Id, + APIMode: apiMode, + ModelConfig: mdl.Config, + VisualProviderId: mdl.VisualProviderId, + VisualModelKey: mdl.VisualModelKey, + ProjectId: vk.ProjectId, + DomainId: vk.DomainId, + RoutingLog: resolved.routingLog, } if vk.Limits != nil { up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest @@ -368,14 +373,11 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, } // ResolveVisualUpstream resolves the visual provider upstream for tool-delegated image analysis. -func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, cfg *api.SAiModelVisualConfig) (*ChatUpstream, error) { - if cfg == nil || !cfg.Enabled { - return nil, errors.Wrap(httperrors.ErrInputParameter, "visual extension is not enabled") - } - providerID := strings.TrimSpace(cfg.VisualAiProviderId) - modelKey := strings.TrimSpace(cfg.VisualModelKey) +func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, visualProviderId, visualModelKey string) (*ChatUpstream, error) { + providerID := strings.TrimSpace(visualProviderId) + modelKey := strings.TrimSpace(visualModelKey) if providerID == "" || modelKey == "" { - return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_ai_provider_id and visual_model_key are required") + return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required") } pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerID) if err != nil { diff --git a/pkg/aiproxy/models/list_models.go b/pkg/aiproxy/models/list_models.go index 9881c19322..210066886b 100644 --- a/pkg/aiproxy/models/list_models.go +++ b/pkg/aiproxy/models/list_models.go @@ -210,6 +210,40 @@ func ClientFacingModelIDsForRouting( return out } +// VisualActiveClientModelIDsForRouting returns client-facing model ids whose +// bound ai_model has VisualActive() (enabled visual extension + provider/model columns). +func VisualActiveClientModelIDsForRouting( + routing *SAiRouting, + bindings []SAiRoutingModel, + modelsById map[string]*SAiModel, + providers map[string]*SAiProvider, +) map[string]struct{} { + out := make(map[string]struct{}) + if routing == nil { + return out + } + routeKey := strings.TrimSpace(routing.ModelKey) + for i := range bindings { + e := &bindings[i] + prov := providers[e.AiProviderId] + mdl := modelsById[e.AiModelId] + if prov == nil || mdl == nil || !mdl.VisualActive() { + continue + } + var id string + if routeKey != "" { + id = hierarchicalClientModelID(routing, e, mdl) + } else { + id = clientFacingModelID(routing, e, mdl) + } + if id == "" { + continue + } + out[id] = struct{}{} + } + return out +} + func hierarchicalClientModelID(routing *SAiRouting, entry *SAiRoutingModel, mdl *SAiModel) string { if routing == nil { return "" @@ -300,10 +334,8 @@ func modelsListEntryFromModel(id, ownedBy string, created int64, mdl *SAiModel) Created: created, OwnedBy: ownedBy, } - if mdl != nil && mdl.Config != nil { - if mods := mdl.Config.InputModalitiesForCatalog(); len(mods) > 0 { - entry.InputModalities = mods - } + if mdl != nil && mdl.VisualActive() { + entry.InputModalities = []string{"text", "image"} } return entry } diff --git a/pkg/aiproxy/models/list_models_test.go b/pkg/aiproxy/models/list_models_test.go index 42a91bc4ac..28b19f6842 100644 --- a/pkg/aiproxy/models/list_models_test.go +++ b/pkg/aiproxy/models/list_models_test.go @@ -14,7 +14,11 @@ package models -import "testing" +import ( + "testing" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) func TestClientFacingModelID(t *testing.T) { mdl := &SAiModel{ModelKey: "gpt-4o-mini"} @@ -71,6 +75,45 @@ func TestClientFacingModelIDsForRouting(t *testing.T) { } } +func TestVisualActiveClientModelIDsForRouting(t *testing.T) { + routing := &SAiRouting{ModelKey: "test-model"} + active := &SAiModel{ + ModelKey: "deepseek-v4-flash", + VisualProviderId: "moonshot-id", + VisualModelKey: "moonshot-v1-8k-vision-preview", + Config: &api.SAiModelConfig{ + Extensions: &api.SAiModelExtensions{ + Visual: &api.SAiModelVisualConfig{Enabled: true}, + }, + }, + } + inactive := &SAiModel{ModelKey: "deepseek-v4-pro"} + prov := &SAiProvider{ProviderKey: "deepseek"} + bindings := []SAiRoutingModel{ + {AiProviderId: "p1", AiModelId: "m-active"}, + {AiProviderId: "p1", AiModelId: "m-inactive"}, + } + modelsById := map[string]*SAiModel{ + "m-active": active, + "m-inactive": inactive, + } + providers := map[string]*SAiProvider{"p1": prov} + + got := VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers) + if len(got) != 1 { + t.Fatalf("len = %d, want 1: %#v", len(got), got) + } + if _, ok := got["test-model/deepseek-v4-flash"]; !ok { + t.Fatalf("missing hierarchical visual id: %#v", got) + } + if _, ok := got["test-model/deepseek-v4-pro"]; ok { + t.Fatal("inactive visual model should not be included") + } + if len(VisualActiveClientModelIDsForRouting(nil, bindings, modelsById, providers)) != 0 { + t.Fatal("nil routing should yield empty set") + } +} + func TestHierarchicalClientModelID(t *testing.T) { routing := &SAiRouting{ModelKey: "claude"} mdl := &SAiModel{ModelKey: "claude-sonnet-4-6"} diff --git a/pkg/aiproxy/providers/openai/responses_compat_test.go b/pkg/aiproxy/providers/openai/responses_compat_test.go index 244aa2ba04..e5d31024a1 100644 --- a/pkg/aiproxy/providers/openai/responses_compat_test.go +++ b/pkg/aiproxy/providers/openai/responses_compat_test.go @@ -234,3 +234,58 @@ func TestResponsesStreamConverterText(t *testing.T) { t.Fatalf("events = %+v", end) } } + +func TestNonStreamChatCompletionToStreamPayloads(t *testing.T) { + raw := []byte(`{ + "id":"chatcmpl-visual-1", + "model":"deepseek-v4-flash", + "choices":[{ + "message":{ + "role":"assistant", + "reasoning_content":"thinking", + "content":"Hello from visual" + }, + "finish_reason":"stop" + }], + "usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15} + }`) + payloads, err := NonStreamChatCompletionToStreamPayloads(raw) + if err != nil { + t.Fatal(err) + } + if len(payloads) < 3 { + t.Fatalf("payloads = %d, want at least 3", len(payloads)) + } + + conv := NewResponsesStreamConverter("deepseek-v4-flash", nil) + var events []ResponsesStreamEvent + for _, payload := range payloads { + chunkEvents, err := conv.Feed(payload, false) + if err != nil { + t.Fatal(err) + } + events = append(events, chunkEvents...) + } + endEvents, err := conv.Feed(nil, true) + if err != nil { + t.Fatal(err) + } + events = append(events, endEvents...) + + foundCreated := false + foundText := false + foundCompleted := false + for _, e := range events { + switch e.Event { + case "response.created": + foundCreated = true + case "response.output_text.delta": + foundText = true + case "response.completed": + foundCompleted = true + } + } + if !foundCreated || !foundText || !foundCompleted { + t.Fatalf("events missing lifecycle: created=%v text=%v completed=%v all=%+v", foundCreated, foundText, foundCompleted, events) + } +} diff --git a/pkg/aiproxy/providers/openai/responses_stream_compat.go b/pkg/aiproxy/providers/openai/responses_stream_compat.go index ceed85bf27..6936a0b7c8 100644 --- a/pkg/aiproxy/providers/openai/responses_stream_compat.go +++ b/pkg/aiproxy/providers/openai/responses_stream_compat.go @@ -434,3 +434,135 @@ func ToProviderChunks(events []ResponsesStreamEvent) []providerapi.ResponsesStre } return out } + +type syntheticChatChunkDelta struct { + Content *string `json:"content,omitempty"` + ReasoningContent *string `json:"reasoning_content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +type syntheticChatChunkChoice struct { + Delta syntheticChatChunkDelta `json:"delta,omitempty"` + FinishReason string `json:"finish_reason,omitempty"` +} + +type syntheticChatChunkPayload struct { + ID string `json:"id,omitempty"` + Model string `json:"model,omitempty"` + Choices []syntheticChatChunkChoice `json:"choices,omitempty"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage,omitempty"` +} + +// NonStreamChatCompletionToStreamPayloads converts a chat.completion JSON body into +// synthetic chat.completion.chunk payloads for ResponsesStreamConverter. +func NonStreamChatCompletionToStreamPayloads(body []byte) ([][]byte, error) { + var resp struct { + ID string `json:"id"` + Model string `json:"model"` + Choices []struct { + Message struct { + Content json.RawMessage `json:"content"` + ReasoningContent string `json:"reasoning_content"` + ToolCalls []ToolCall `json:"tool_calls"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("invalid chat completion: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("empty chat completion choices") + } + choice := resp.Choices[0] + payloads := make([][]byte, 0, 4) + + appendChunk := func(chunk syntheticChatChunkPayload) error { + if chunk.ID == "" { + chunk.ID = resp.ID + } + if chunk.Model == "" { + chunk.Model = resp.Model + } + b, err := json.Marshal(chunk) + if err != nil { + return err + } + payloads = append(payloads, b) + return nil + } + + if err := appendChunk(syntheticChatChunkPayload{ + ID: resp.ID, + Model: resp.Model, + Choices: []syntheticChatChunkChoice{{}}, + }); err != nil { + return nil, err + } + + if rc := strings.TrimSpace(choice.Message.ReasoningContent); rc != "" { + reason := rc + if err := appendChunk(syntheticChatChunkPayload{ + Choices: []syntheticChatChunkChoice{{ + Delta: syntheticChatChunkDelta{ReasoningContent: &reason}, + }}, + }); err != nil { + return nil, err + } + } + + if text := strings.TrimSpace(MessageTextContent(choice.Message.Content)); text != "" { + content := text + if err := appendChunk(syntheticChatChunkPayload{ + Choices: []syntheticChatChunkChoice{{ + Delta: syntheticChatChunkDelta{Content: &content}, + }}, + }); err != nil { + return nil, err + } + } + + for i, tc := range choice.Message.ToolCalls { + call := tc + if call.Index == 0 && i > 0 { + call.Index = i + } + if call.Type == "" { + call.Type = "function" + } + if err := appendChunk(syntheticChatChunkPayload{ + Choices: []syntheticChatChunkChoice{{ + Delta: syntheticChatChunkDelta{ToolCalls: []ToolCall{call}}, + }}, + }); err != nil { + return nil, err + } + } + + if resp.Usage != nil { + if err := appendChunk(syntheticChatChunkPayload{Usage: resp.Usage}); err != nil { + return nil, err + } + } + + if choice.FinishReason == "length" { + if err := appendChunk(syntheticChatChunkPayload{ + Choices: []syntheticChatChunkChoice{{ + FinishReason: "length", + }}, + }); err != nil { + return nil, err + } + } + + return payloads, nil +} diff --git a/pkg/apis/aiproxy/ai_model.go b/pkg/apis/aiproxy/ai_model.go index 288d4bf03e..2d1e9a1374 100644 --- a/pkg/apis/aiproxy/ai_model.go +++ b/pkg/apis/aiproxy/ai_model.go @@ -31,27 +31,36 @@ type AiModelListInput struct { type AiModelCreateInput struct { apis.EnabledStatusStandaloneResourceCreateInput - AiProviderId string `json:"ai_provider_id"` - ModelKey string `json:"model_key"` - Config *SAiModelConfig `json:"config"` + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` + VisualProviderId string `json:"visual_provider_id"` + VisualModelKey string `json:"visual_model_key"` + Config *SAiModelConfig `json:"config"` } type AiModelUpdateInput struct { apis.EnabledStatusStandaloneResourceBaseUpdateInput - AiProviderId string `json:"ai_provider_id"` - ModelKey string `json:"model_key"` - Enabled *bool `json:"enabled"` - Config *SAiModelConfig `json:"config"` + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` + VisualProviderId string `json:"visual_provider_id"` + VisualModelKey string `json:"visual_model_key"` + Enabled *bool `json:"enabled"` + Config *SAiModelConfig `json:"config"` } type AiModelDetails struct { apis.EnabledStatusStandaloneResourceDetails - AiProviderId string `json:"ai_provider_id"` - AiProviderName string `json:"ai_provider_name"` - ModelKey string `json:"model_key"` - Config *SAiModelConfig `json:"config"` + AiProviderId string `json:"ai_provider_id"` + AiProviderName string `json:"ai_provider_name"` + ModelKey string `json:"model_key"` + VisualProviderId string `json:"visual_provider_id"` + VisualProviderName string `json:"visual_provider_name"` + VisualProviderKey string `json:"visual_provider_key"` + VisualModelKey string `json:"visual_model_key"` + VisualActive bool `json:"visual_active"` + Config *SAiModelConfig `json:"config"` } // SAiModelConfig stores per-model extension settings. @@ -64,12 +73,11 @@ type SAiModelExtensions struct { } // SAiModelVisualConfig enables tool-delegated vision for text-only upstream models. +// visual_provider_id / visual_model_key live on ai_model columns, not in this JSON. type SAiModelVisualConfig struct { - Enabled bool `json:"enabled"` - VisualAiProviderId string `json:"visual_ai_provider_id"` - VisualModelKey string `json:"visual_model_key"` - MaxRounds int `json:"max_rounds,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` + Enabled bool `json:"enabled"` + MaxRounds int `json:"max_rounds,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` } // String implements gotypes.ISerializable for sqlchemy JSON/compound columns. diff --git a/pkg/mcclient/options/aiproxy/resources.go b/pkg/mcclient/options/aiproxy/resources.go index 4759c1bfda..a55f36d1bf 100644 --- a/pkg/mcclient/options/aiproxy/resources.go +++ b/pkg/mcclient/options/aiproxy/resources.go @@ -137,22 +137,42 @@ type AiModelShowOptions struct { type AiModelCreateOptions struct { options.BaseCreateOptions - AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"` - ModelKey string `help:"model routing key" json:"model_key"` - Enabled *bool `json:"enabled,omitempty"` + AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"` + ModelKey string `help:"model routing key" json:"model_key"` + VisualProviderId string `help:"ai_provider for tool-delegated vision" json:"visual_provider_id"` + VisualModelKey string `help:"upstream model id for visual analysis" json:"visual_model_key"` + VisualEnabled *bool `help:"enable visual extension" json:"-"` + VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"` + VisualMaxTokens int `help:"visual analysis max tokens" json:"-"` + Enabled *bool `json:"enabled,omitempty"` } func (o *AiModelCreateOptions) Params() (jsonutils.JSONObject, error) { - return options.StructToParams(o) + params, err := options.StructToParams(o) + if err != nil { + return nil, err + } + params.Remove("visual_enabled") + params.Remove("visual_max_rounds") + params.Remove("visual_max_tokens") + if err := mergeAiModelVisualConfig(params, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, false); err != nil { + return nil, err + } + return params, nil } type AiModelUpdateOptions struct { - ID string `help:"ID or name" json:"-"` - Name string `json:"name,omitempty"` - Desc string `json:"description,omitempty"` - AiProviderId string `json:"ai_provider_id,omitempty"` - ModelKey string `json:"model_key,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + AiProviderId string `json:"ai_provider_id,omitempty"` + ModelKey string `json:"model_key,omitempty"` + VisualProviderId string `json:"visual_provider_id,omitempty"` + VisualModelKey string `json:"visual_model_key,omitempty"` + VisualEnabled *bool `help:"enable visual extension" json:"-"` + VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"` + VisualMaxTokens int `help:"visual analysis max tokens" json:"-"` + Enabled *bool `json:"enabled,omitempty"` } func (o *AiModelUpdateOptions) GetId() string { @@ -160,7 +180,45 @@ func (o *AiModelUpdateOptions) GetId() string { } func (o *AiModelUpdateOptions) Params() (jsonutils.JSONObject, error) { - return options.StructToParams(o) + d, err := options.StructToParams(o) + if err != nil { + return nil, err + } + d.Remove("visual_enabled") + d.Remove("visual_max_rounds") + d.Remove("visual_max_tokens") + if err := mergeAiModelVisualConfig(d, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, true); err != nil { + return nil, err + } + return d, nil +} + +// mergeAiModelVisualConfig builds config.extensions.visual from flat CLI flags. +// requireEnabled=true for update: max_* alone would replace the whole config and drop enabled. +func mergeAiModelVisualConfig(params *jsonutils.JSONDict, enabled *bool, maxRounds, maxTokens int, requireEnabled bool) error { + hasMax := maxRounds > 0 || maxTokens > 0 + if enabled == nil && !hasMax { + return nil + } + if requireEnabled && enabled == nil && hasMax { + return errors.Errorf("set --visual-enabled when using --visual-max-rounds or --visual-max-tokens (config is replaced as a whole)") + } + visual := jsonutils.NewDict() + if enabled != nil { + visual.Set("enabled", jsonutils.NewBool(*enabled)) + } + if maxRounds > 0 { + visual.Set("max_rounds", jsonutils.NewInt(int64(maxRounds))) + } + if maxTokens > 0 { + visual.Set("max_tokens", jsonutils.NewInt(int64(maxTokens))) + } + extensions := jsonutils.NewDict() + extensions.Set("visual", visual) + config := jsonutils.NewDict() + config.Set("extensions", extensions) + params.Set("config", config) + return nil } type AiModelDeleteOptions struct {