fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(llm): keep aiproxy routing across stop/restart (#25673)

Browse Source 
Do not unsync or delete deployment ai_routing when replicas leave running; reconcile on running so routing/provider IDs stay stable.
This commit is contained in:
Zexi Li
2026-09-11 16:43:18 +08:00
committed by GitHub
parent 3fd882452b
commit d203580430
3 changed files with 81 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/llm/models/llm.go
View File

@@ -569,15 +569,12 @@ func (llm *SLLM) SetStatus(ctx context.Context, userCred mcclient.TokenCredentia
if err := dep.SyncReadyReplicas(ctx, userCred); err != nil {
log.Warningf("SLLM.SetStatus: SyncReadyReplicas for deployment %s: %s", llm.LLMDeploymentId, err)
}
if dep.AutoRegisterAiproxy {
if status == api.LLM_STATUS_RUNNING {
if err := dep.StartAiproxySyncTask(ctx, userCred, llm.Id, ""); err != nil {
log.Warningf("SLLM.SetStatus: start aiproxy sync for llm %s: %v", llm.Id, err)
}
} else if oldStatus == api.LLM_STATUS_RUNNING && status != api.LLM_STATUS_RUNNING {
if err := UnsyncLlmInstance(ctx, userCred, dep, llm.Id); err != nil {
log.Warningf("SLLM.SetStatus: unsync aiproxy for llm %s: %v", llm.Id, err)
}
// Restart/stop must not delete aiproxy routing or providers; catalog is
// removed only on deployment delete or manual unregister. Reconcile on
// running upserts the same routing/provider IDs (e.g. new pod URL).
if dep.AutoRegisterAiproxy && status == api.LLM_STATUS_RUNNING {
if err := dep.StartAiproxySyncTask(ctx, userCred, llm.Id, ""); err != nil {
log.Warningf("SLLM.SetStatus: start aiproxy sync for llm %s: %v", llm.Id, err)
}
}
return nil

40
pkg/llm/models/llm_aiproxy_sync.go
View File

@@ -935,7 +935,21 @@ func DeleteDeploymentAiproxyResources(ctx context.Context, deploymentId string)
return nil
}
// UnsyncLlmInstance removes aiproxy resources for one llm replica.
// shouldUnsyncAiproxyOnLeaveRunning reports whether leaving running should
// unbind/delete aiproxy catalog. Restart and stop keep routing IDs stable;
// catalog is only removed on deployment delete or manual unregister.
func shouldUnsyncAiproxyOnLeaveRunning() bool {
return false
}
// unsyncKeepsDeploymentRouting returns the routing id to persist after
// UnsyncLlmInstance. The deployment-level ai_routing row is never deleted here.
func unsyncKeepsDeploymentRouting(existingRoutingId string) (keepRoutingId string, deleteRouting bool) {
return strings.TrimSpace(existingRoutingId), false
}
// UnsyncLlmInstance unbinds one llm replica from the deployment routing.
// It does not delete the deployment's ai_routing or clear AiproxyRoutingId.
func UnsyncLlmInstance(ctx context.Context, userCred mcclient.TokenCredential, dep *SLLMDeployment, llmId string) error {
if dep == nil || strings.TrimSpace(llmId) == "" {
return nil
@@ -953,25 +967,23 @@ func UnsyncLlmInstance(ctx context.Context, userCred mcclient.TokenCredential, d
}
}
routingId := strings.TrimSpace(dep.AiproxyRoutingId)
if routingId == "" {
if err := deleteAiProviderByLlmId(session, llmId); err != nil {
return err
}
if err := persistDeploymentAiproxyBindings(dep, "", nil); err != nil {
return err
}
return dep.SetAiproxySyncStatus(ctx, userCred, api.AIPROXY_SYNC_STATUS_PENDING, "waiting for running replicas")
}
if len(remaining) == 0 {
routingId, deleteRouting := unsyncKeepsDeploymentRouting(dep.AiproxyRoutingId)
if deleteRouting {
if err := deleteAiRoutingById(session, routingId); err != nil {
log.Warningf("delete ai_routing %s: %v", routingId, err)
}
routingId = ""
}
if routingId == "" || len(remaining) == 0 {
if routingId != "" {
if err := applyRoutingModels(session, routingId, []apapi.AiRoutingModelItem{}); err != nil {
log.Warningf("clear ai_routing models %s: %v", routingId, err)
}
}
if err := deleteAiProviderByLlmId(session, llmId); err != nil {
return err
}
if err := persistDeploymentAiproxyBindings(dep, "", nil); err != nil {
if err := persistDeploymentAiproxyBindings(dep, routingId, nil); err != nil {
return err
}
return dep.SetAiproxySyncStatus(ctx, userCred, api.AIPROXY_SYNC_STATUS_PENDING, "waiting for running replicas")

49
pkg/llm/models/llm_aiproxy_sync_test.go Normal file
View File

@@ -0,0 +1,49 @@
package models
import (
"testing"
api "yunion.io/x/onecloud/pkg/apis/llm"
)
func TestShouldUnsyncAiproxyOnLeaveRunning(t *testing.T) {
if shouldUnsyncAiproxyOnLeaveRunning() {
t.Fatal("leaving running must not unsync aiproxy; catalog is removed only on deployment delete or unregister")
}
}
func TestUnsyncKeepsDeploymentRouting(t *testing.T) {
cases := []struct {
name string
in string
want string
}{
{name: "keeps existing id on last replica", in: "routing-abc", want: "routing-abc"},
{name: "trims space", in: " routing-abc ", want: "routing-abc"},
{name: "empty stays empty", in: "", want: ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, deleteRouting := unsyncKeepsDeploymentRouting(tc.in)
if deleteRouting {
t.Fatalf("deleteRouting = true, want false (routing must survive unsync)")
}
if got != tc.want {
t.Fatalf("keepRoutingId = %q, want %q", got, tc.want)
}
})
}
}
func TestAiproxyCatalogDeleteEntrypoints(t *testing.T) {
if api.AIPROXY_SYNC_STATUS_DISABLED == "" {
t.Fatal("disabled status must remain the unregister/delete end state")
}
if shouldUnsyncAiproxyOnLeaveRunning() {
t.Fatal("status leave-running is not a catalog delete entrypoint")
}
_, deleteRouting := unsyncKeepsDeploymentRouting("keep-me")
if deleteRouting {
t.Fatal("UnsyncLlmInstance must not delete deployment routing")
}
}