From 17a8835d59b62eeaa8585d7ae66db15dfdf7512b Mon Sep 17 00:00:00 2001 From: ssongliu Date: Mon, 17 Aug 2026 17:38:43 +0800 Subject: [PATCH] feat: support Ascend 910B GPU monitoring (#13579) * feat: support Ascend 910B GPU monitoring * chore: remove GPU test files --- agent/app/dto/dashboard.go | 1 + agent/app/dto/monitor.go | 1 + agent/app/repo/monitor.go | 3 + agent/app/service/monitor.go | 5 + agent/utils/ai_tools/gpu/ascend.go | 164 ++++++++++++++++++++ agent/utils/ai_tools/gpu/common/gpu_info.go | 1 + agent/utils/ai_tools/gpu/gpu.go | 92 ++++++++++- agent/utils/ai_tools/gpu/schema/parser.go | 1 + frontend/src/api/interface/ai.ts | 2 + frontend/src/api/interface/dashboard.ts | 1 + frontend/src/views/ai/gpu/current/index.vue | 57 +++++-- frontend/src/views/ai/gpu/history/index.vue | 40 ++--- frontend/src/views/home/status/index.vue | 4 +- 13 files changed, 337 insertions(+), 35 deletions(-) create mode 100644 agent/utils/ai_tools/gpu/ascend.go diff --git a/agent/app/dto/dashboard.go b/agent/app/dto/dashboard.go index e216bcba0..c7d83ac2e 100644 --- a/agent/app/dto/dashboard.go +++ b/agent/app/dto/dashboard.go @@ -156,6 +156,7 @@ type DiskInfo struct { } type GPUInfo struct { + Type string `json:"type"` Index uint `json:"index"` ProductName string `json:"productName"` GPUUtil string `json:"gpuUtil"` diff --git a/agent/app/dto/monitor.go b/agent/app/dto/monitor.go index 636bfdd67..6f229adb7 100644 --- a/agent/app/dto/monitor.go +++ b/agent/app/dto/monitor.go @@ -49,6 +49,7 @@ type GPUChartHide struct { GPU bool `json:"gpu"` Memory bool `json:"memory"` Power bool `json:"power"` + PowerLimit bool `json:"powerLimit"` Temperature bool `json:"temperature"` Speed bool `json:"speed"` } diff --git a/agent/app/repo/monitor.go b/agent/app/repo/monitor.go index 64d9c0ec6..381425228 100644 --- a/agent/app/repo/monitor.go +++ b/agent/app/repo/monitor.go @@ -73,6 +73,9 @@ func (u *MonitorRepo) CreateMonitorBase(model model.MonitorBase) error { return global.MonitorDB.Create(&model).Error } func (s *MonitorRepo) BatchCreateMonitorGPU(list []model.MonitorGPU) error { + if len(list) == 0 { + return nil + } return global.GPUMonitorDB.CreateInBatches(&list, len(list)).Error } func (u *MonitorRepo) BatchCreateMonitorIO(ioList []model.MonitorIO) error { diff --git a/agent/app/service/monitor.go b/agent/app/service/monitor.go index 5a1c4a8d7..98bd166f8 100644 --- a/agent/app/service/monitor.go +++ b/agent/app/service/monitor.go @@ -155,6 +155,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions { if (item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A") && (item.PowerDraw == "" || item.PowerDraw == "N/A") { chartHide.Power = true } + chartHide.PowerLimit = item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A" chartHide.Temperature = item.Temperature == "" || item.Temperature == "N/A" chartHide.Speed = item.FanSpeed == "" || item.FanSpeed == "N/A" data.ChartHide = append(data.ChartHide, chartHide) @@ -174,6 +175,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions { var chartHide dto.GPUChartHide chartHide.GPU = true chartHide.Speed = true + chartHide.PowerLimit = true chartHide.ProductName = fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName) if (item.Stats.MemoryUsed == "" || item.Stats.MemoryUsed == "N/A") && (item.Basic.Memory == "" || item.Basic.FreeMemory == "N/A") { chartHide.Memory = true @@ -344,6 +346,7 @@ func (m *MonitorService) Run() { _ = monitorRepo.DelMonitorBase(timeForDelete) _ = monitorRepo.DelMonitorIO(timeForDelete) _ = monitorRepo.DelMonitorNet(timeForDelete) + _ = monitorRepo.DelMonitorGPU(timeForDelete) } func (m *MonitorService) loadDiskIO() { @@ -599,6 +602,7 @@ func saveGPUDataToDB() { } gpuInfo, err := client.LoadGpuInfo() if err != nil { + global.LOG.Errorf("load gpu monitor data failed, err: %v", err) return } var list []model.MonitorGPU @@ -631,6 +635,7 @@ func saveXPUDataToDB() { } xpuInfo, err := client.LoadGpuInfo() if err != nil { + global.LOG.Errorf("load xpu monitor data failed, err: %v", err) return } var list []model.MonitorGPU diff --git a/agent/utils/ai_tools/gpu/ascend.go b/agent/utils/ai_tools/gpu/ascend.go new file mode 100644 index 000000000..be58da4e6 --- /dev/null +++ b/agent/utils/ai_tools/gpu/ascend.go @@ -0,0 +1,164 @@ +package gpu + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common" + "github.com/1Panel-dev/1Panel/agent/utils/cmd" +) + +var ( + ascendVersionPattern = regexp.MustCompile(`(?i)\bVersion:\s*([^\s|]+)`) + ascendMemoryPattern = regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)`) +) + +type AscendSMI struct{} + +func (a AscendSMI) LoadGpuInfo() (*common.GpuInfo, error) { + cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) + itemData, err := cmdMgr.RunWithStdout("npu-smi", "info") + if err != nil { + return nil, fmt.Errorf("calling npu-smi failed, %v", err) + } + return parseAscendSMI(itemData), nil +} + +func parseAscendSMI(data string) *common.GpuInfo { + info := &common.GpuInfo{Type: "ascend"} + if match := ascendVersionPattern.FindStringSubmatch(data); len(match) == 2 { + info.DriverVersion = match[1] + } + + processSection := false + deviceIndexes := make(map[uint]int) + var pending *common.GPU + + for _, line := range strings.Split(data, "\n") { + if strings.Contains(line, "Process id") && strings.Contains(line, "Process memory") { + processSection = true + pending = nil + continue + } + + cells := ascendTableCells(line) + if processSection { + if len(cells) != 4 && len(cells) != 5 { + continue + } + deviceFields := strings.Fields(cells[0]) + if len(deviceFields) == 0 { + continue + } + deviceID, err := strconv.ParseUint(deviceFields[0], 10, 64) + if err != nil { + continue + } + index, ok := deviceIndexes[uint(deviceID)] + if !ok { + continue + } + processOffset := len(cells) - 3 + info.GPUs[index].Processes = append(info.GPUs[index].Processes, common.Process{ + Pid: strings.TrimSpace(cells[processOffset]), + Type: "NPU", + ProcessName: strings.TrimSpace(cells[processOffset+1]), + UsedMemory: ascendValueWithUnit(cells[processOffset+2], "MB"), + }) + continue + } + + if len(cells) != 3 { + continue + } + if pending == nil { + fields := strings.Fields(cells[0]) + if len(fields) < 2 { + continue + } + deviceID, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + metrics := strings.Fields(cells[2]) + if len(metrics) < 2 { + continue + } + pending = &common.GPU{ + Type: "ascend", + Index: uint(deviceID), + ProductName: strings.Join(fields[1:], " "), + PersistenceMode: "N/A", + DisplayActive: "N/A", + ECC: "N/A", + FanSpeed: "N/A", + Temperature: ascendValueWithUnit(metrics[1], "C"), + PerformanceState: strings.TrimSpace(cells[1]), + PowerDraw: ascendValueWithUnit(metrics[0], "W"), + MaxPowerLimit: "N/A", + ComputeMode: "N/A", + MigMode: "N/A", + } + continue + } + + metrics := strings.Fields(cells[2]) + if len(metrics) == 0 { + pending = nil + continue + } + pending.BusID = strings.TrimSpace(cells[1]) + pending.GPUUtil = ascendValueWithUnit(metrics[0], "%") + pending.MemUsed, pending.MemTotal = ascendMemoryUsage(cells[2]) + deviceIndexes[pending.Index] = len(info.GPUs) + info.GPUs = append(info.GPUs, *pending) + pending = nil + } + + return info +} + +func ascendTableCells(line string) []string { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") { + return nil + } + parts := strings.Split(line, "|") + if len(parts) < 3 { + return nil + } + cells := make([]string, 0, len(parts)-2) + for _, part := range parts[1 : len(parts)-1] { + cells = append(cells, strings.TrimSpace(part)) + } + return cells +} + +func ascendMemoryUsage(value string) (string, string) { + matches := ascendMemoryPattern.FindAllStringSubmatch(value, -1) + if len(matches) == 0 { + return "N/A", "N/A" + } + // 910B exposes both generic memory and HBM. Prefer the last memory pool + // with a non-zero capacity, which selects HBM while retaining compatibility + // with devices that only expose Memory-Usage. + selected := matches[len(matches)-1] + for i := len(matches) - 1; i >= 0; i-- { + if total, err := strconv.ParseFloat(matches[i][2], 64); err == nil && total > 0 { + selected = matches[i] + break + } + } + return selected[1] + " MB", selected[2] + " MB" +} + +func ascendValueWithUnit(value, unit string) string { + value = strings.TrimSpace(value) + if value == "" || strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") { + return "N/A" + } + return value + " " + unit +} diff --git a/agent/utils/ai_tools/gpu/common/gpu_info.go b/agent/utils/ai_tools/gpu/common/gpu_info.go index 3d827eb40..af80dc5fe 100644 --- a/agent/utils/ai_tools/gpu/common/gpu_info.go +++ b/agent/utils/ai_tools/gpu/common/gpu_info.go @@ -9,6 +9,7 @@ type GpuInfo struct { } type GPU struct { + Type string `json:"type"` Index uint `json:"index"` ProductName string `json:"productName"` PersistenceMode string `json:"persistenceMode"` diff --git a/agent/utils/ai_tools/gpu/gpu.go b/agent/utils/ai_tools/gpu/gpu.go index a6ab841a8..340ffaf36 100644 --- a/agent/utils/ai_tools/gpu/gpu.go +++ b/agent/utils/ai_tools/gpu/gpu.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "strings" + "sync" "time" "github.com/1Panel-dev/1Panel/agent/global" @@ -18,8 +19,95 @@ import ( type NvidiaSMI struct{} -func New() (bool, NvidiaSMI) { - return cmd.Which("nvidia-smi"), NvidiaSMI{} +type SMI interface { + LoadGpuInfo() (*common.GpuInfo, error) +} + +func New() (bool, SMI) { + var clients []SMI + if cmd.Which("nvidia-smi") { + clients = append(clients, NvidiaSMI{}) + } + if cmd.Which("npu-smi") { + clients = append(clients, AscendSMI{}) + } + if len(clients) == 0 { + return false, nil + } + if len(clients) == 1 { + return true, clients[0] + } + return true, multiSMI{clients: clients} +} + +type multiSMI struct { + clients []SMI +} + +type smiResult struct { + info *common.GpuInfo + err error +} + +func (m multiSMI) LoadGpuInfo() (*common.GpuInfo, error) { + results := make([]smiResult, len(m.clients)) + var wg sync.WaitGroup + for index, client := range m.clients { + wg.Add(1) + go func() { + defer wg.Done() + results[index].info, results[index].err = client.LoadGpuInfo() + }() + } + wg.Wait() + + merged := &common.GpuInfo{} + var ( + errs []error + types []string + driverVersions []string + ) + for _, result := range results { + if result.err != nil { + errs = append(errs, result.err) + continue + } + if result.info == nil { + continue + } + deviceType := result.info.Type + if deviceType != "" { + types = append(types, deviceType) + } + if result.info.DriverVersion != "" { + driverVersions = append(driverVersions, fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), result.info.DriverVersion)) + } + if result.info.CudaVersion != "" { + merged.CudaVersion = result.info.CudaVersion + } + for _, device := range result.info.GPUs { + if device.Type == "" { + device.Type = deviceType + } + merged.GPUs = append(merged.GPUs, device) + } + } + + if len(types) == 1 { + merged.Type = types[0] + } else if len(types) > 1 { + merged.Type = "mixed" + } + if len(driverVersions) == 1 { + parts := strings.SplitN(driverVersions[0], ": ", 2) + merged.DriverVersion = parts[len(parts)-1] + } else { + merged.DriverVersion = strings.Join(driverVersions, "；") + } + if len(merged.GPUs) == 0 && len(errs) > 0 { + return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...)) + } + return merged, nil } func (n NvidiaSMI) LoadGpuInfo() (*common.GpuInfo, error) { diff --git a/agent/utils/ai_tools/gpu/schema/parser.go b/agent/utils/ai_tools/gpu/schema/parser.go index 10efe1c19..a80cdf541 100644 --- a/agent/utils/ai_tools/gpu/schema/parser.go +++ b/agent/utils/ai_tools/gpu/schema/parser.go @@ -23,6 +23,7 @@ func Parse(buf []byte, version string) (*common.GpuInfo, error) { } for i := 0; i < len(s.Gpu); i++ { var gpuItem common.GPU + gpuItem.Type = "nvidia" gpuItem.Index = uint(i) gpuItem.ProductName = s.Gpu[i].ProductName gpuItem.PersistenceMode = s.Gpu[i].PersistenceMode diff --git a/frontend/src/api/interface/ai.ts b/frontend/src/api/interface/ai.ts index 1f7b835a3..75ba639ed 100644 --- a/frontend/src/api/interface/ai.ts +++ b/frontend/src/api/interface/ai.ts @@ -28,6 +28,7 @@ export namespace AI { gpu: GPU[]; } export interface GPU { + type: string; index: number; productName: string; persistenceMode: string; @@ -63,6 +64,7 @@ export namespace AI { gpu: boolean; memory: boolean; power: boolean; + powerLimit: boolean; temperature: boolean; speed: boolean; } diff --git a/frontend/src/api/interface/dashboard.ts b/frontend/src/api/interface/dashboard.ts index 74e58213c..97b8c66bc 100644 --- a/frontend/src/api/interface/dashboard.ts +++ b/frontend/src/api/interface/dashboard.ts @@ -144,6 +144,7 @@ export namespace Dashboard { inodesUsedPercent: number; } export interface GPUInfo { + type: string; index: number; productName: string; gpuUtil: string; diff --git a/frontend/src/views/ai/gpu/current/index.vue b/frontend/src/views/ai/gpu/current/index.vue index 1269580a0..01b7a2e2f 100644 --- a/frontend/src/views/ai/gpu/current/index.vue +++ b/frontend/src/views/ai/gpu/current/index.vue @@ -1,7 +1,7 @@