fix(host): use assigned nvidia GPU memory for pod metrics (#25608)

Derive mem total from isolated-device quotas when available, and harden
losetup ListDevices against empty --json output.

pkg/hostman/hostinfo/container.go

@@ -17,6 +17,7 @@ package hostinfo
 import (
 	"context"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -100,6 +101,38 @@ func (h *SHostInfo) GetNvidiaGpuIndexMemoryMap() map[string]int {
 	return res
 }
 
+func (h *SHostInfo) GetNvidiaGpuIndexByDeviceIds(ids []string) map[string]int {
+	h.HasContainerNvidiaGpu()
+	want := make(map[string]struct{}, len(ids))
+	for _, id := range ids {
+		if id != "" {
+			want[id] = struct{}{}
+		}
+	}
+	res := map[string]int{}
+	for i := range h.containerNvidiaGpus {
+		iDev := h.containerNvidiaGpus[i]
+		cloudId := iDev.GetCloudId()
+		if _, ok := want[cloudId]; !ok {
+			continue
+		}
+		if _, exists := res[cloudId]; exists {
+			continue
+		}
+		nv, ok := iDev.(INvidiaGpuIndexMemoryInterface)
+		if !ok {
+			continue
+		}
+		idx, err := strconv.Atoi(nv.GetNvidiaDevIndex())
+		if err != nil {
+			log.Errorf("failed parse nvidia gpu index %s: %s", nv.GetNvidiaDevIndex(), err)
+			continue
+		}
+		res[cloudId] = idx
+	}
+	return res
+}
+
 func (h *SHostInfo) HasContainerVastaitechGpu() bool {
 	if h.hasVastaitechGpus != nil {
 		return *h.hasVastaitechGpus

pkg/hostman/hostmetrics/container_metrics.go

@@ -62,6 +62,7 @@ const (
 	NVIDIA_GPU_FRAME_BUFFER   = "frame_buffer"
 	NVIDIA_GPU_CCPM           = "ccpm"
 	NVIDIA_GPU_SM             = "sm"
+	NVIDIA_GPU_MEM            = "mem"
 	NVIDIA_GPU_MEM_UTIL       = "mem_util"
 	NVIDIA_GPU_ENC            = "enc"
 	NVIDIA_GPU_DEC            = "dec"
@@ -249,6 +250,7 @@ func (m PodNvidiaGpuMetrics) ToMap() map[string]interface{} {
 		NVIDIA_GPU_FRAME_BUFFER:   m.Framebuffer,
 		NVIDIA_GPU_CCPM:           m.Ccpm,
 		NVIDIA_GPU_SM:             m.SmUtil,
+		NVIDIA_GPU_MEM:            m.Mem,
 		NVIDIA_GPU_MEM_UTIL:       m.MemUtil,
 		NVIDIA_GPU_ENC:            m.EncUtil,
 		NVIDIA_GPU_DEC:            m.DecUtil,
@@ -890,16 +892,62 @@ func (m *SGuestMonitor) getPodVastaitechGpuMetrics() []*PodVastaitechGpuMetrics
 	return res
 }
 
-func (m *SGuestMonitor) getPodNvidiaGpuMetrics() []*PodNvidiaGpuMetrics {
-	if len(m.nvidiaGpuMetrics) == 0 {
-		return nil
+type nvidiaGpuAssignedQuota struct {
+	PhysicalIndex int
+	MemTotal      int
+}
+
+type nvidiaGpuAssignInput struct {
+	Id          string
+	MemoryLimit int
+}
+
+func buildNvidiaGpuAssignedQuotas(inputs []nvidiaGpuAssignInput, indexById map[string]int, hostMemMap map[string]int) []nvidiaGpuAssignedQuota {
+	type acc struct {
+		limitSum int
+		limitCnt int
+		hostMem  int
 	}
+	accs := map[int]*acc{}
+	order := make([]int, 0)
+	for _, in := range inputs {
+		idx, ok := indexById[in.Id]
+		if !ok {
+			continue
+		}
+		a, exists := accs[idx]
+		if !exists {
+			a = &acc{hostMem: hostMemMap[strconv.Itoa(idx)]}
+			accs[idx] = a
+			order = append(order, idx)
+		}
+		if in.MemoryLimit > 0 {
+			a.limitSum += in.MemoryLimit
+			a.limitCnt++
+		}
+	}
+	res := make([]nvidiaGpuAssignedQuota, 0, len(order))
+	for _, idx := range order {
+		a := accs[idx]
+		memTotal := a.hostMem
+		if a.limitCnt > 0 {
+			memTotal = a.limitSum
+		}
+		res = append(res, nvidiaGpuAssignedQuota{
+			PhysicalIndex: idx,
+			MemTotal:      memTotal,
+		})
+	}
+	return res
+}
+
+func (m *SGuestMonitor) getPodNvidiaGpuMetrics() []*PodNvidiaGpuMetrics {
 	indexGpuMap := map[int]*PodNvidiaGpuMetrics{}
 	for i := range m.nvidiaGpuMetrics {
 		index := m.nvidiaGpuMetrics[i].Index
 		gms, ok := indexGpuMap[index]
 		if !ok {
-			gms = new(PodNvidiaGpuMetrics)
+			gms = &PodNvidiaGpuMetrics{Index: index}
 		}
 		gms.Framebuffer += m.nvidiaGpuMetrics[i].FB
 		gms.Ccpm += m.nvidiaGpuMetrics[i].Ccpm
@@ -911,17 +959,33 @@ func (m *SGuestMonitor) getPodNvidiaGpuMetrics() []*PodNvidiaGpuMetrics {
 		indexGpuMap[index] = gms
 	}
 
-	indexs := make([]int, 0)
+	assignedMem := map[int]int{}
+	for _, a := range m.nvidiaGpuAssigned {
+		assignedMem[a.PhysicalIndex] = a.MemTotal
+		if _, ok := indexGpuMap[a.PhysicalIndex]; !ok {
+			indexGpuMap[a.PhysicalIndex] = &PodNvidiaGpuMetrics{Index: a.PhysicalIndex}
+		}
+	}
+
+	if len(indexGpuMap) == 0 {
+		return nil
+	}
+
+	indexs := make([]int, 0, len(indexGpuMap))
 	for index, gms := range indexGpuMap {
 		indexs = append(indexs, index)
-		indexStr := strconv.Itoa(index)
-		memSizeTotal, ok := m.nvidiaGpuIndexMemoryMap[indexStr]
+		memSizeTotal, ok := assignedMem[index]
 		if !ok {
-			continue
+			memSizeTotal, ok = m.nvidiaGpuIndexMemoryMap[strconv.Itoa(index)]
+			if !ok {
+				continue
+			}
 		}
 		gms.MemTotal = memSizeTotal
 		gms.Mem = gms.Framebuffer
-		gms.MemUtil = float64(gms.Framebuffer) / float64(gms.MemTotal)
+		if gms.MemTotal > 0 {
+			gms.MemUtil = float64(gms.Framebuffer) / float64(gms.MemTotal)
+		}
 	}
 	sort.Ints(indexs)
 	res := make([]*PodNvidiaGpuMetrics, len(indexs))

pkg/hostman/hostmetrics/container_metrics_nvidia_gpu_test.go

@@ -0,0 +1,172 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"testing"
+)
+
+func TestBuildNvidiaGpuAssignedQuotasHamiUsesMemoryLimit(t *testing.T) {
+	got := buildNvidiaGpuAssignedQuotas(
+		[]nvidiaGpuAssignInput{{Id: "dev-hami", MemoryLimit: 8192}},
+		map[string]int{"dev-hami": 0},
+		map[string]int{"0": 81920},
+	)
+	if len(got) != 1 {
+		t.Fatalf("len(got)=%d, want 1", len(got))
+	}
+	if got[0].PhysicalIndex != 0 {
+		t.Errorf("PhysicalIndex=%d, want 0", got[0].PhysicalIndex)
+	}
+	if got[0].MemTotal != 8192 {
+		t.Errorf("MemTotal=%d, want 8192 (HAMI slice)", got[0].MemTotal)
+	}
+}
+
+func TestBuildNvidiaGpuAssignedQuotasMpsUsesHostMap(t *testing.T) {
+	got := buildNvidiaGpuAssignedQuotas(
+		[]nvidiaGpuAssignInput{{Id: "dev-mps", MemoryLimit: 0}},
+		map[string]int{"dev-mps": 1},
+		map[string]int{"1": 10240},
+	)
+	if len(got) != 1 {
+		t.Fatalf("len(got)=%d, want 1", len(got))
+	}
+	if got[0].PhysicalIndex != 1 {
+		t.Errorf("PhysicalIndex=%d, want 1", got[0].PhysicalIndex)
+	}
+	if got[0].MemTotal != 10240 {
+		t.Errorf("MemTotal=%d, want 10240 (MPS slice from host map)", got[0].MemTotal)
+	}
+}
+
+func TestBuildNvidiaGpuAssignedQuotasSameGpuSumsMemoryLimit(t *testing.T) {
+	got := buildNvidiaGpuAssignedQuotas(
+		[]nvidiaGpuAssignInput{
+			{Id: "dev-a", MemoryLimit: 4096},
+			{Id: "dev-b", MemoryLimit: 4096},
+		},
+		map[string]int{"dev-a": 0, "dev-b": 0},
+		map[string]int{"0": 81920},
+	)
+	if len(got) != 1 {
+		t.Fatalf("len(got)=%d, want 1", len(got))
+	}
+	if got[0].MemTotal != 8192 {
+		t.Errorf("MemTotal=%d, want 8192 (sum of MemoryLimit)", got[0].MemTotal)
+	}
+}
+
+func TestGetPodNvidiaGpuMetricsIdleAssignedZeroFill(t *testing.T) {
+	m := &SGuestMonitor{
+		nvidiaGpuAssigned: []nvidiaGpuAssignedQuota{
+			{PhysicalIndex: 2, MemTotal: 8192},
+		},
+		nvidiaGpuIndexMemoryMap: map[string]int{"2": 81920},
+	}
+	got := m.getPodNvidiaGpuMetrics()
+	if len(got) != 1 {
+		t.Fatalf("len(got)=%d, want 1", len(got))
+	}
+	if got[0].PhysicalIndex != 2 {
+		t.Errorf("PhysicalIndex=%d, want 2", got[0].PhysicalIndex)
+	}
+	if got[0].Index != 0 {
+		t.Errorf("Index=%d, want 0 (pod-local)", got[0].Index)
+	}
+	if got[0].MemTotal != 8192 {
+		t.Errorf("MemTotal=%d, want 8192", got[0].MemTotal)
+	}
+	if got[0].Mem != 0 || got[0].MemUtil != 0 {
+		t.Errorf("Mem=%d MemUtil=%v, want 0", got[0].Mem, got[0].MemUtil)
+	}
+}
+
+func TestGetPodNvidiaGpuMetricsProcessPlusIdleAssigned(t *testing.T) {
+	m := &SGuestMonitor{
+		nvidiaGpuMetrics: []NvidiaGpuProcessMetrics{
+			{Index: 0, FB: 1024},
+		},
+		nvidiaGpuAssigned: []nvidiaGpuAssignedQuota{
+			{PhysicalIndex: 0, MemTotal: 8192},
+			{PhysicalIndex: 1, MemTotal: 8192},
+		},
+	}
+	got := m.getPodNvidiaGpuMetrics()
+	if len(got) != 2 {
+		t.Fatalf("len(got)=%d, want 2", len(got))
+	}
+	if got[0].PhysicalIndex != 0 || got[0].Index != 0 {
+		t.Errorf("gpu0 PhysicalIndex=%d Index=%d, want 0, 0", got[0].PhysicalIndex, got[0].Index)
+	}
+	if got[0].Mem != 1024 {
+		t.Errorf("gpu0 Mem=%d, want 1024", got[0].Mem)
+	}
+	if got[0].MemUtil != 1024.0/8192.0 {
+		t.Errorf("gpu0 MemUtil=%v, want %v", got[0].MemUtil, 1024.0/8192.0)
+	}
+	if got[1].PhysicalIndex != 1 || got[1].Index != 1 {
+		t.Errorf("gpu1 PhysicalIndex=%d Index=%d, want 1, 1", got[1].PhysicalIndex, got[1].Index)
+	}
+	if got[1].Mem != 0 || got[1].MemUtil != 0 {
+		t.Errorf("gpu1 Mem=%d MemUtil=%v, want 0", got[1].Mem, got[1].MemUtil)
+	}
+}
+
+func TestGetPodNvidiaGpuMetricsHamiSliceUtil(t *testing.T) {
+	m := &SGuestMonitor{
+		nvidiaGpuMetrics: []NvidiaGpuProcessMetrics{
+			{Index: 0, FB: 8192},
+		},
+		nvidiaGpuAssigned: []nvidiaGpuAssignedQuota{
+			{PhysicalIndex: 0, MemTotal: 8192},
+		},
+		nvidiaGpuIndexMemoryMap: map[string]int{"0": 81920},
+	}
+	got := m.getPodNvidiaGpuMetrics()
+	if len(got) != 1 {
+		t.Fatalf("len(got)=%d, want 1", len(got))
+	}
+	if got[0].MemTotal != 8192 {
+		t.Errorf("MemTotal=%d, want 8192 (HAMI slice, not full card)", got[0].MemTotal)
+	}
+	if got[0].Mem != 8192 {
+		t.Errorf("Mem=%d, want 8192", got[0].Mem)
+	}
+	if got[0].MemUtil != 1 {
+		t.Errorf("MemUtil=%v, want 1", got[0].MemUtil)
+	}
+	if got[0].PhysicalIndex != 0 {
+		t.Errorf("PhysicalIndex=%d, want 0", got[0].PhysicalIndex)
+	}
+}
+
+func TestGetPodNvidiaGpuMetricsNilWithoutAssignedOrProcess(t *testing.T) {
+	m := &SGuestMonitor{}
+	if got := m.getPodNvidiaGpuMetrics(); got != nil {
+		t.Errorf("got %v, want nil", got)
+	}
+}
+
+func TestPodNvidiaGpuMetricsToMapIncludesMem(t *testing.T) {
+	m := PodNvidiaGpuMetrics{Mem: 512, MemUtil: 0.25, MemTotal: 2048}
+	got := m.ToMap()
+	if got[NVIDIA_GPU_MEM] != 512 {
+		t.Errorf("mem=%v, want 512", got[NVIDIA_GPU_MEM])
+	}
+	if got[NVIDIA_GPU_MEM_UTIL] != 0.25 {
+		t.Errorf("mem_util=%v, want 0.25", got[NVIDIA_GPU_MEM_UTIL])
+	}
+}

pkg/hostman/hostmetrics/hostmetrics.go

@@ -65,6 +65,7 @@ type IHostInfo interface {
 	HasContainerVastaitechGpu() bool
 	HasContainerCphAmdGpu() bool
 	GetNvidiaGpuIndexMemoryMap() map[string]int
+	GetNvidiaGpuIndexByDeviceIds(ids []string) map[string]int
 	ReportHostDmesg(data []compute.SKmsgEntry) error
 }
 
@@ -629,6 +630,7 @@ type SGuestMonitor struct {
 	podStat                 *stats.PodStats
 	nvidiaGpuMetrics        []NvidiaGpuProcessMetrics
 	nvidiaGpuIndexMemoryMap map[string]int
+	nvidiaGpuAssigned       []nvidiaGpuAssignedQuota
 	vastaitechGpuMetrics    []VastaitechGpuProcessMetrics
 	cphAmdGpuMetrics        []CphAmdGpuProcessMetrics
 	instance                guestman.GuestRuntimeInstance
@@ -660,6 +662,7 @@ func NewGuestPodMonitor(
 	hasNvGpu := false
 	hasCphAmdGpu := false
 	hasVastaitechGpu := false
+	nvAssignInputs := make([]nvidiaGpuAssignInput, 0)
 	for i := range podDesc.IsolatedDevices {
 		if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) {
 			continue
@@ -668,6 +671,10 @@ func NewGuestPodMonitor(
 		switch vendorId {
 		case compute.NVIDIA_VENDOR_ID:
 			hasNvGpu = true
+			nvAssignInputs = append(nvAssignInputs, nvidiaGpuAssignInput{
+				Id:          podDesc.IsolatedDevices[i].Id,
+				MemoryLimit: podDesc.IsolatedDevices[i].MemoryLimit,
+			})
 		case compute.AMD_VENDOR_ID:
 			hasCphAmdGpu = true
 		case compute.VASTAITECH_VENDOR_ID:
@@ -678,6 +685,12 @@ func NewGuestPodMonitor(
 	if hasNvGpu {
 		m.nvidiaGpuMetrics = GetPodNvidiaGpuMetrics(nvidiaGpuMetrics, podProcs)
 		m.nvidiaGpuIndexMemoryMap = hostInstance.GetNvidiaGpuIndexMemoryMap()
+		nvDevIds := make([]string, 0, len(nvAssignInputs))
+		for i := range nvAssignInputs {
+			nvDevIds = append(nvDevIds, nvAssignInputs[i].Id)
+		}
+		indexById := hostInstance.GetNvidiaGpuIndexByDeviceIds(nvDevIds)
+		m.nvidiaGpuAssigned = buildNvidiaGpuAssignedQuotas(nvAssignInputs, indexById, m.nvidiaGpuIndexMemoryMap)
 	}
 	if hasVastaitechGpu {
 		m.vastaitechGpuMetrics = GetPodVastaitechGpuMetrics(vastaitechGpuMetrics, podProcs)

pkg/util/losetup/losetup.go

@@ -81,6 +81,9 @@ func NewLosetupCommand() *LosetupCommand {
 }
 
 func parseJsonOutput(content string) (*Devices, error) {
+	if strings.TrimSpace(content) == "" {
+		return &Devices{}, nil
+	}
 	obj, err := jsonutils.ParseString(content)
 	if err != nil {
 		return nil, errors.Wrapf(err, "parse json: %s", content)
@@ -93,20 +96,28 @@ func parseJsonOutput(content string) (*Devices, error) {
 }
 
 func ListDevices() (*Devices, error) {
-	cmd, err := NewLosetupCommand().AddArgs("--json").Run()
+	cmd, err := NewLosetupCommand().AddArgs("--json", "-l").Run()
+	if err == nil {
+		output := cmd.Output()
+		if strings.TrimSpace(output) != "" {
+			devs, parseErr := parseJsonOutput(output)
+			if parseErr == nil {
+				return devs, nil
+			}
+			err = parseErr
+		}
+	}
+
 	errs := make([]error, 0)
 	if err != nil {
 		errs = append(errs, errors.Wrap(err, "list by json"))
-		devs, err2 := listDevicesOldVersion()
-		if err2 != nil {
-			errs = append(errs, errors.Wrap(err, "list by using old way"))
-		} else {
-			return devs, nil
-		}
+	}
+	devs, err2 := listDevicesOldVersion()
+	if err2 != nil {
+		errs = append(errs, errors.Wrap(err2, "list by using old way"))
 		return nil, errors.NewAggregate(errs)
 	}
-	output := cmd.Output()
-	return parseJsonOutput(output)
+	return devs, nil
 }
 
 func listDevicesOldVersion() (*Devices, error) {

pkg/util/losetup/losetup_test.go

@@ -67,6 +67,18 @@ func Test_parseJsonOutput(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name:    "empty string",
+			content: "",
+			want:    &Devices{},
+			wantErr: false,
+		},
+		{
+			name:    "whitespace only",
+			content: " \n\t",
+			want:    &Devices{},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {