fix(llm): skip stale container status during pod transitions (#25611)

Watch container events and defer LLM status updates while the pod is starting/stopping, so exited leftovers are not treated as start_fail.

pkg/llm/models/llm_pod_status_watcher.go

@@ -27,6 +27,10 @@ func StartLLMPodStatusWatcher(ctx context.Context, region string) {
 		if err := watchMan.For(compute.Servers).AddEventHandler(ctx, handler); err != nil {
 			return errors.Wrap(err, "watch compute servers for llm pod status")
 		}
+		ctrHandler := &llmContainerStatusEventHandler{podHandler: handler}
+		if err := watchMan.For(compute.Containers).AddEventHandler(ctx, ctrHandler); err != nil {
+			return errors.Wrap(err, "watch compute containers for llm pod status")
+		}
 		return nil
 	})
 }
@@ -58,6 +62,13 @@ func (h *llmPodStatusEventHandler) handleServerStatus(ctx context.Context, obj *
 		log.Warningf("LLM pod status watcher: server event missing id: %s", obj.String())
 		return
 	}
+	h.handleByServerId(ctx, serverId)
+}
+
+func (h *llmPodStatusEventHandler) handleByServerId(ctx context.Context, serverId string) {
+	if serverId == "" {
+		return
+	}
 
 	llm, err := fetchLLMByCmpId(serverId)
 	if err != nil {
@@ -89,6 +100,36 @@ func (h *llmPodStatusEventHandler) handleServerStatus(ctx context.Context, obj *
 	}
 }
 
+type llmContainerStatusEventHandler struct {
+	podHandler *llmPodStatusEventHandler
+}
+
+func (h *llmContainerStatusEventHandler) OnAdd(obj *jsonutils.JSONDict) {
+	if !containerWatchStatusChanged(nil, obj) {
+		return
+	}
+	h.handleContainerStatus(context.Background(), obj)
+}
+
+func (h *llmContainerStatusEventHandler) OnUpdate(oldObj, newObj *jsonutils.JSONDict) {
+	if !containerWatchStatusChanged(oldObj, newObj) {
+		return
+	}
+	h.handleContainerStatus(context.Background(), newObj)
+}
+
+func (h *llmContainerStatusEventHandler) OnDelete(obj *jsonutils.JSONDict) {
+}
+
+func (h *llmContainerStatusEventHandler) handleContainerStatus(ctx context.Context, obj *jsonutils.JSONDict) {
+	guestId := watchEventStringField(obj, "guest_id")
+	if guestId == "" {
+		log.Warningf("LLM container status watcher: event missing guest_id: %s", obj.String())
+		return
+	}
+	h.podHandler.handleByServerId(ctx, guestId)
+}
+
 func fetchLLMByCmpId(cmpId string) (*SLLM, error) {
 	llm := &SLLM{}
 	if err := GetLLMManager().Query().Equals("cmp_id", cmpId).First(llm); err != nil {
@@ -129,6 +170,20 @@ func serverWatchStatusChanged(oldObj *jsonutils.JSONDict, newObj *jsonutils.JSON
 	return watchEventContainerStatusSignature(oldObj) != watchEventContainerStatusSignature(newObj)
 }
 
+func containerWatchStatusChanged(oldObj *jsonutils.JSONDict, newObj *jsonutils.JSONDict) bool {
+	if newObj == nil {
+		return false
+	}
+	newStatus := watchEventStringField(newObj, "status")
+	if newStatus == "" {
+		return false
+	}
+	if oldObj == nil {
+		return true
+	}
+	return watchEventStringField(oldObj, "status") != newStatus
+}
+
 func watchEventStringField(obj *jsonutils.JSONDict, key string) string {
 	if obj == nil {
 		return ""

pkg/llm/models/llm_pod_status_watcher_test.go

@@ -0,0 +1,34 @@
+package models
+
+import (
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestContainerWatchStatusChanged(t *testing.T) {
+	probing := jsonutils.NewDict()
+	probing.Set("status", jsonutils.NewString("probing"))
+	probing.Set("guest_id", jsonutils.NewString("srv-1"))
+
+	running := jsonutils.NewDict()
+	running.Set("status", jsonutils.NewString("running"))
+	running.Set("guest_id", jsonutils.NewString("srv-1"))
+
+	if !containerWatchStatusChanged(nil, probing) {
+		t.Fatal("add with status should trigger")
+	}
+	if containerWatchStatusChanged(probing, probing) {
+		t.Fatal("same status should not trigger")
+	}
+	if !containerWatchStatusChanged(probing, running) {
+		t.Fatal("probing -> running should trigger")
+	}
+	if containerWatchStatusChanged(running, nil) {
+		t.Fatal("nil new object should not trigger")
+	}
+	empty := jsonutils.NewDict()
+	if containerWatchStatusChanged(probing, empty) {
+		t.Fatal("missing status should not trigger")
+	}
+}

pkg/llm/models/llm_status_resolver.go

@@ -32,6 +32,12 @@ func resolveLLMStatusFromPod(currentStatus string, serverStatus string, primaryC
 	targetStatus := ""
 	reason := fmt.Sprintf("pod status=%s primary_container_status=%s", serverStatus, primaryContainerStatus)
 
+	// Guest is still starting/stopping; container status is often stale (e.g. exited
+	// from the previous shutdown while pod is start_start). Wait for a terminal state.
+	if isLLMPodTransientStatus(serverStatus) {
+		return llmStatusResolution{}
+	}
+
 	switch {
 	case serverStatus == computeapi.VM_READY:
 		targetStatus = api.LLM_STATUS_READY
@@ -64,6 +70,21 @@ func resolveLLMStatusFromPod(currentStatus string, serverStatus string, primaryC
 	}
 }
 
+func isLLMPodTransientStatus(serverStatus string) bool {
+	switch serverStatus {
+	case computeapi.VM_START_START,
+		computeapi.VM_STARTING,
+		computeapi.POD_STATUS_STARTING_CONTAINER,
+		computeapi.POD_STATUS_CREATING_CONTAINER,
+		computeapi.VM_START_STOP,
+		computeapi.VM_STOPPING,
+		computeapi.POD_STATUS_STOPPING_CONTAINER:
+		return true
+	default:
+		return false
+	}
+}
+
 func isLLMPodCrashLoopStatus(serverStatus string, primaryContainerStatus string) bool {
 	if isPrimaryContainerRunning(primaryContainerStatus) {
 		return false

pkg/llm/models/llm_status_resolver_test.go

@@ -87,6 +87,27 @@ func TestResolveLLMStatusFromPod(t *testing.T) {
 			wantStatus:             api.LLM_STATUS_START_FAIL,
 			wantUpdate:             true,
 		},
+		{
+			name:                   "start_start with stale exited is not start_fail",
+			currentStatus:          api.LLM_STATUS_READY,
+			serverStatus:           computeapi.VM_START_START,
+			primaryContainerStatus: computeapi.CONTAINER_STATUS_EXITED,
+			wantUpdate:             false,
+		},
+		{
+			name:                   "starting_container with stale exited is not start_fail",
+			currentStatus:          api.LLM_STATUS_READY,
+			serverStatus:           computeapi.POD_STATUS_STARTING_CONTAINER,
+			primaryContainerStatus: computeapi.CONTAINER_STATUS_EXITED,
+			wantUpdate:             false,
+		},
+		{
+			name:                   "start_start with stale probe_failed is not start_fail",
+			currentStatus:          api.LLM_STATUS_READY,
+			serverStatus:           computeapi.VM_START_START,
+			primaryContainerStatus: computeapi.CONTAINER_STATUS_PROBE_FAILED,
+			wantUpdate:             false,
+		},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {