feat: support Ascend 910B GPU monitoring (#13579)

* feat: support Ascend 910B GPU monitoring

* chore: remove GPU test files

agent/app/dto/dashboard.go

@@ -156,6 +156,7 @@ type DiskInfo struct {
 }
 
 type GPUInfo struct {
+	Type             string `json:"type"`
 	Index            uint   `json:"index"`
 	ProductName      string `json:"productName"`
 	GPUUtil          string `json:"gpuUtil"`

agent/app/dto/monitor.go

@@ -49,6 +49,7 @@ type GPUChartHide struct {
 	GPU         bool   `json:"gpu"`
 	Memory      bool   `json:"memory"`
 	Power       bool   `json:"power"`
+	PowerLimit  bool   `json:"powerLimit"`
 	Temperature bool   `json:"temperature"`
 	Speed       bool   `json:"speed"`
 }

agent/app/repo/monitor.go

@@ -73,6 +73,9 @@ func (u *MonitorRepo) CreateMonitorBase(model model.MonitorBase) error {
 	return global.MonitorDB.Create(&model).Error
 }
 func (s *MonitorRepo) BatchCreateMonitorGPU(list []model.MonitorGPU) error {
+	if len(list) == 0 {
+		return nil
+	}
 	return global.GPUMonitorDB.CreateInBatches(&list, len(list)).Error
 }
 func (u *MonitorRepo) BatchCreateMonitorIO(ioList []model.MonitorIO) error {

agent/app/service/monitor.go

@@ -155,6 +155,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
 			if (item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A") && (item.PowerDraw == "" || item.PowerDraw == "N/A") {
 				chartHide.Power = true
 			}
+			chartHide.PowerLimit = item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A"
 			chartHide.Temperature = item.Temperature == "" || item.Temperature == "N/A"
 			chartHide.Speed = item.FanSpeed == "" || item.FanSpeed == "N/A"
 			data.ChartHide = append(data.ChartHide, chartHide)
@@ -174,6 +175,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
 			var chartHide dto.GPUChartHide
 			chartHide.GPU = true
 			chartHide.Speed = true
+			chartHide.PowerLimit = true
 			chartHide.ProductName = fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName)
 			if (item.Stats.MemoryUsed == "" || item.Stats.MemoryUsed == "N/A") && (item.Basic.Memory == "" || item.Basic.FreeMemory == "N/A") {
 				chartHide.Memory = true
@@ -344,6 +346,7 @@ func (m *MonitorService) Run() {
 	_ = monitorRepo.DelMonitorBase(timeForDelete)
 	_ = monitorRepo.DelMonitorIO(timeForDelete)
 	_ = monitorRepo.DelMonitorNet(timeForDelete)
+	_ = monitorRepo.DelMonitorGPU(timeForDelete)
 }
 
 func (m *MonitorService) loadDiskIO() {
@@ -599,6 +602,7 @@ func saveGPUDataToDB() {
 	}
 	gpuInfo, err := client.LoadGpuInfo()
 	if err != nil {
+		global.LOG.Errorf("load gpu monitor data failed, err: %v", err)
 		return
 	}
 	var list []model.MonitorGPU
@@ -631,6 +635,7 @@ func saveXPUDataToDB() {
 	}
 	xpuInfo, err := client.LoadGpuInfo()
 	if err != nil {
+		global.LOG.Errorf("load xpu monitor data failed, err: %v", err)
 		return
 	}
 	var list []model.MonitorGPU

agent/utils/ai_tools/gpu/ascend.go

@@ -0,0 +1,164 @@
+package gpu
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common"
+	"github.com/1Panel-dev/1Panel/agent/utils/cmd"
+)
+
+var (
+	ascendVersionPattern = regexp.MustCompile(`(?i)\bVersion:\s*([^\s|]+)`)
+	ascendMemoryPattern  = regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)`)
+)
+
+type AscendSMI struct{}
+
+func (a AscendSMI) LoadGpuInfo() (*common.GpuInfo, error) {
+	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second))
+	itemData, err := cmdMgr.RunWithStdout("npu-smi", "info")
+	if err != nil {
+		return nil, fmt.Errorf("calling npu-smi failed, %v", err)
+	}
+	return parseAscendSMI(itemData), nil
+}
+
+func parseAscendSMI(data string) *common.GpuInfo {
+	info := &common.GpuInfo{Type: "ascend"}
+	if match := ascendVersionPattern.FindStringSubmatch(data); len(match) == 2 {
+		info.DriverVersion = match[1]
+	}
+
+	processSection := false
+	deviceIndexes := make(map[uint]int)
+	var pending *common.GPU
+
+	for _, line := range strings.Split(data, "\n") {
+		if strings.Contains(line, "Process id") && strings.Contains(line, "Process memory") {
+			processSection = true
+			pending = nil
+			continue
+		}
+
+		cells := ascendTableCells(line)
+		if processSection {
+			if len(cells) != 4 && len(cells) != 5 {
+				continue
+			}
+			deviceFields := strings.Fields(cells[0])
+			if len(deviceFields) == 0 {
+				continue
+			}
+			deviceID, err := strconv.ParseUint(deviceFields[0], 10, 64)
+			if err != nil {
+				continue
+			}
+			index, ok := deviceIndexes[uint(deviceID)]
+			if !ok {
+				continue
+			}
+			processOffset := len(cells) - 3
+			info.GPUs[index].Processes = append(info.GPUs[index].Processes, common.Process{
+				Pid:         strings.TrimSpace(cells[processOffset]),
+				Type:        "NPU",
+				ProcessName: strings.TrimSpace(cells[processOffset+1]),
+				UsedMemory:  ascendValueWithUnit(cells[processOffset+2], "MB"),
+			})
+			continue
+		}
+
+		if len(cells) != 3 {
+			continue
+		}
+		if pending == nil {
+			fields := strings.Fields(cells[0])
+			if len(fields) < 2 {
+				continue
+			}
+			deviceID, err := strconv.ParseUint(fields[0], 10, 64)
+			if err != nil {
+				continue
+			}
+			metrics := strings.Fields(cells[2])
+			if len(metrics) < 2 {
+				continue
+			}
+			pending = &common.GPU{
+				Type:             "ascend",
+				Index:            uint(deviceID),
+				ProductName:      strings.Join(fields[1:], " "),
+				PersistenceMode:  "N/A",
+				DisplayActive:    "N/A",
+				ECC:              "N/A",
+				FanSpeed:         "N/A",
+				Temperature:      ascendValueWithUnit(metrics[1], "C"),
+				PerformanceState: strings.TrimSpace(cells[1]),
+				PowerDraw:        ascendValueWithUnit(metrics[0], "W"),
+				MaxPowerLimit:    "N/A",
+				ComputeMode:      "N/A",
+				MigMode:          "N/A",
+			}
+			continue
+		}
+
+		metrics := strings.Fields(cells[2])
+		if len(metrics) == 0 {
+			pending = nil
+			continue
+		}
+		pending.BusID = strings.TrimSpace(cells[1])
+		pending.GPUUtil = ascendValueWithUnit(metrics[0], "%")
+		pending.MemUsed, pending.MemTotal = ascendMemoryUsage(cells[2])
+		deviceIndexes[pending.Index] = len(info.GPUs)
+		info.GPUs = append(info.GPUs, *pending)
+		pending = nil
+	}
+
+	return info
+}
+
+func ascendTableCells(line string) []string {
+	line = strings.TrimSpace(line)
+	if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") {
+		return nil
+	}
+	parts := strings.Split(line, "|")
+	if len(parts) < 3 {
+		return nil
+	}
+	cells := make([]string, 0, len(parts)-2)
+	for _, part := range parts[1 : len(parts)-1] {
+		cells = append(cells, strings.TrimSpace(part))
+	}
+	return cells
+}
+
+func ascendMemoryUsage(value string) (string, string) {
+	matches := ascendMemoryPattern.FindAllStringSubmatch(value, -1)
+	if len(matches) == 0 {
+		return "N/A", "N/A"
+	}
+	// 910B exposes both generic memory and HBM. Prefer the last memory pool
+	// with a non-zero capacity, which selects HBM while retaining compatibility
+	// with devices that only expose Memory-Usage.
+	selected := matches[len(matches)-1]
+	for i := len(matches) - 1; i >= 0; i-- {
+		if total, err := strconv.ParseFloat(matches[i][2], 64); err == nil && total > 0 {
+			selected = matches[i]
+			break
+		}
+	}
+	return selected[1] + " MB", selected[2] + " MB"
+}
+
+func ascendValueWithUnit(value, unit string) string {
+	value = strings.TrimSpace(value)
+	if value == "" || strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") {
+		return "N/A"
+	}
+	return value + " " + unit
+}

agent/utils/ai_tools/gpu/common/gpu_info.go

@@ -9,6 +9,7 @@ type GpuInfo struct {
 }
 
 type GPU struct {
+	Type            string `json:"type"`
 	Index           uint   `json:"index"`
 	ProductName     string `json:"productName"`
 	PersistenceMode string `json:"persistenceMode"`

agent/utils/ai_tools/gpu/gpu.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/1Panel-dev/1Panel/agent/global"
@@ -18,8 +19,95 @@ import (
 
 type NvidiaSMI struct{}
 
-func New() (bool, NvidiaSMI) {
+type SMI interface {
-	return cmd.Which("nvidia-smi"), NvidiaSMI{}
+	LoadGpuInfo() (*common.GpuInfo, error)
+}
+
+func New() (bool, SMI) {
+	var clients []SMI
+	if cmd.Which("nvidia-smi") {
+		clients = append(clients, NvidiaSMI{})
+	}
+	if cmd.Which("npu-smi") {
+		clients = append(clients, AscendSMI{})
+	}
+	if len(clients) == 0 {
+		return false, nil
+	}
+	if len(clients) == 1 {
+		return true, clients[0]
+	}
+	return true, multiSMI{clients: clients}
+}
+
+type multiSMI struct {
+	clients []SMI
+}
+
+type smiResult struct {
+	info *common.GpuInfo
+	err  error
+}
+
+func (m multiSMI) LoadGpuInfo() (*common.GpuInfo, error) {
+	results := make([]smiResult, len(m.clients))
+	var wg sync.WaitGroup
+	for index, client := range m.clients {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			results[index].info, results[index].err = client.LoadGpuInfo()
+		}()
+	}
+	wg.Wait()
+
+	merged := &common.GpuInfo{}
+	var (
+		errs           []error
+		types          []string
+		driverVersions []string
+	)
+	for _, result := range results {
+		if result.err != nil {
+			errs = append(errs, result.err)
+			continue
+		}
+		if result.info == nil {
+			continue
+		}
+		deviceType := result.info.Type
+		if deviceType != "" {
+			types = append(types, deviceType)
+		}
+		if result.info.DriverVersion != "" {
+			driverVersions = append(driverVersions, fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), result.info.DriverVersion))
+		}
+		if result.info.CudaVersion != "" {
+			merged.CudaVersion = result.info.CudaVersion
+		}
+		for _, device := range result.info.GPUs {
+			if device.Type == "" {
+				device.Type = deviceType
+			}
+			merged.GPUs = append(merged.GPUs, device)
+		}
+	}
+
+	if len(types) == 1 {
+		merged.Type = types[0]
+	} else if len(types) > 1 {
+		merged.Type = "mixed"
+	}
+	if len(driverVersions) == 1 {
+		parts := strings.SplitN(driverVersions[0], ": ", 2)
+		merged.DriverVersion = parts[len(parts)-1]
+	} else {
+		merged.DriverVersion = strings.Join(driverVersions, "；")
+	}
+	if len(merged.GPUs) == 0 && len(errs) > 0 {
+		return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...))
+	}
+	return merged, nil
 }
 
 func (n NvidiaSMI) LoadGpuInfo() (*common.GpuInfo, error) {

agent/utils/ai_tools/gpu/schema/parser.go

@@ -23,6 +23,7 @@ func Parse(buf []byte, version string) (*common.GpuInfo, error) {
 	}
 	for i := 0; i < len(s.Gpu); i++ {
 		var gpuItem common.GPU
+		gpuItem.Type = "nvidia"
 		gpuItem.Index = uint(i)
 		gpuItem.ProductName = s.Gpu[i].ProductName
 		gpuItem.PersistenceMode = s.Gpu[i].PersistenceMode

frontend/src/api/interface/ai.ts

@@ -28,6 +28,7 @@ export namespace AI {
         gpu: GPU[];
     }
     export interface GPU {
+        type: string;
         index: number;
         productName: string;
         persistenceMode: string;
@@ -63,6 +64,7 @@ export namespace AI {
         gpu: boolean;
         memory: boolean;
         power: boolean;
+        powerLimit: boolean;
         temperature: boolean;
         speed: boolean;
     }

frontend/src/api/interface/dashboard.ts

@@ -144,6 +144,7 @@ export namespace Dashboard {
         inodesUsedPercent: number;
     }
     export interface GPUInfo {
+        type: string;
         index: number;
         productName: string;
         gpuUtil: string;

frontend/src/views/ai/gpu/current/index.vue

@@ -1,7 +1,7 @@
 <template>
     <div>
         <RouterMenu />
-        <div v-if="gpuType == 'nvidia'">
+        <div v-if="gpuType !== 'xpu'">
             <LayoutContent
                 v-loading="loading"
                 :title="$t('aiTools.gpu.gpu')"
@@ -14,15 +14,27 @@
                 </template>
                 <template #main>
                     <el-descriptions direction="vertical" :column="14" border>
-                        <el-descriptions-item :label="$t('aiTools.gpu.driverVersion')" width="50%" :span="7">
+                        <el-descriptions-item
+                            :label="$t('aiTools.gpu.driverVersion')"
+                            width="50%"
+                            :span="gpuInfo.cudaVersion ? 7 : 14"
+                        >
                             {{ gpuInfo.driverVersion }}
                         </el-descriptions-item>
-                        <el-descriptions-item :label="$t('aiTools.gpu.cudaVersion')" :span="7">
+                        <el-descriptions-item
+                            v-if="gpuInfo.cudaVersion"
+                            :label="$t('aiTools.gpu.cudaVersion')"
+                            :span="7"
+                        >
                             {{ gpuInfo.cudaVersion }}
                         </el-descriptions-item>
                     </el-descriptions>
                     <el-collapse v-model="activeNames" class="card-interval">
-                        <el-collapse-item v-for="item in gpuInfo.gpu" :key="item.index" :name="item.index">
+                        <el-collapse-item
+                            v-for="item in gpuInfo.gpu"
+                            :key="item.type + '-' + item.index"
+                            :name="item.type + '-' + item.index"
+                        >
                             <template #title>
                                 <span class="name-class">{{ item.index + '. ' + item.productName }}</span>
                             </template>
@@ -45,8 +57,13 @@
                                 <el-descriptions-item>
                                     <template #label>
                                         <div class="cell-item">
-                                            {{ $t('aiTools.gpu.performanceState') }}
+                                            {{
+                                                item.type === 'ascend'
+                                                    ? $t('commons.table.status')
+                                                    : $t('aiTools.gpu.performanceState')
+                                            }}
                                             <el-tooltip
+                                                v-if="item.type === 'nvidia'"
                                                 placement="top"
                                                 :content="$t('aiTools.gpu.performanceStateHelper')"
                                             >
@@ -57,19 +74,20 @@
                                     {{ item.performanceState }}
                                 </el-descriptions-item>
                                 <el-descriptions-item :label="$t('aiTools.gpu.powerUsage')">
-                                    {{ item.powerDraw }} / {{ item.maxPowerLimit }}
+                                    {{ item.powerDraw }}
+                                    <template v-if="item.maxPowerLimit !== 'N/A'">/ {{ item.maxPowerLimit }}</template>
                                 </el-descriptions-item>
                                 <el-descriptions-item :label="$t('aiTools.gpu.memoryUsage')">
                                     {{ item.memUsed }} / {{ item.memTotal }}
                                 </el-descriptions-item>
-                                <el-descriptions-item :label="$t('aiTools.gpu.fanSpeed')">
+                                <el-descriptions-item v-if="item.type === 'nvidia'" :label="$t('aiTools.gpu.fanSpeed')">
                                     {{ item.fanSpeed }}
                                 </el-descriptions-item>
 
                                 <el-descriptions-item :label="$t('aiTools.gpu.busID')">
                                     {{ item.busID }}
                                 </el-descriptions-item>
-                                <el-descriptions-item>
+                                <el-descriptions-item v-if="item.type === 'nvidia'">
                                     <template #label>
                                         <div class="cell-item">
                                             {{ $t('aiTools.gpu.persistenceMode') }}
@@ -83,14 +101,17 @@
                                     </template>
                                     {{ $t('aiTools.gpu.' + item.persistenceMode.toLowerCase()) }}
                                 </el-descriptions-item>
-                                <el-descriptions-item :label="$t('aiTools.gpu.displayActive')">
+                                <el-descriptions-item
+                                    v-if="item.type === 'nvidia'"
+                                    :label="$t('aiTools.gpu.displayActive')"
+                                >
                                     {{
                                         lowerCase(item.displayActive) === 'disabled'
                                             ? $t('aiTools.gpu.displayActiveF')
                                             : $t('aiTools.gpu.displayActiveT')
                                     }}
                                 </el-descriptions-item>
-                                <el-descriptions-item>
+                                <el-descriptions-item v-if="item.type === 'nvidia'">
                                     <template #label>
                                         <div class="cell-item">
                                             Uncorr. ECC
@@ -101,7 +122,10 @@
                                     </template>
                                     {{ loadEcc(item.ecc) }}
                                 </el-descriptions-item>
-                                <el-descriptions-item :label="$t('aiTools.gpu.computeMode')">
+                                <el-descriptions-item
+                                    v-if="item.type === 'nvidia'"
+                                    :label="$t('aiTools.gpu.computeMode')"
+                                >
                                     <template #label>
                                         <div class="cell-item">
                                             {{ $t('aiTools.gpu.computeMode') }}
@@ -121,7 +145,7 @@
                                     </template>
                                     {{ loadComputeMode(item.computeMode) }}
                                 </el-descriptions-item>
-                                <el-descriptions-item label="MIG.M">
+                                <el-descriptions-item v-if="item.type === 'nvidia'" label="MIG.M">
                                     <template #label>
                                         <div class="cell-item">
                                             MIG M.
@@ -268,7 +292,7 @@ const search = async () => {
         .then((res) => {
             loading.value = false;
             gpuType.value = res.data.type;
-            if (res.data.type == 'nvidia') {
+            if (res.data.type !== 'xpu') {
                 gpuInfo.value = res.data;
             } else {
                 xpuInfo.value = res.data;
@@ -281,7 +305,12 @@ const search = async () => {
 
 const refresh = async () => {
     const res = await loadGPUInfo();
-    gpuInfo.value = res.data;
+    gpuType.value = res.data.type;
+    if (res.data.type !== 'xpu') {
+        gpuInfo.value = res.data;
+    } else {
+        xpuInfo.value = res.data;
+    }
 };
 
 const lowerCase = (val: string) => {

frontend/src/views/ai/gpu/history/index.vue

@@ -319,13 +319,15 @@ function initMemoryCharts(baseDate: any, data: any) {
 }
 
 function initPowerCharts(baseDate: any, data: any) {
-    chartsOption.value['loadPowerChart'] = {
+    const yData: any[] = [
-        xData: baseDate,
+        {
-        yData: [
+            name: i18n.global.t('aiTools.gpu.powerCurrent'),
-            {
+            data: data.powerUsed,
-                name: i18n.global.t('aiTools.gpu.powerCurrent'),
+        },
-                data: data.powerUsed,
+    ];
-            },
+    const yAxis: any[] = [{ type: 'value', name: i18n.global.t('aiTools.gpu.power') }];
+    if (!currentHide.value?.powerLimit) {
+        yData.push(
             {
                 name: i18n.global.t('aiTools.gpu.powerLimit'),
                 data: data.powerTotal,
@@ -335,16 +337,18 @@ function initPowerCharts(baseDate: any, data: any) {
                 data: data.powerPercent,
                 yAxisIndex: 1,
             },
-        ],
+        );
-        yAxis: [
+        yAxis.push({
-            { type: 'value', name: i18n.global.t('aiTools.gpu.power') },
+            type: 'value',
-            {
+            name: i18n.global.t('aiTools.gpu.percent') + ' ( % )',
-                type: 'value',
+            position: 'right',
-                name: i18n.global.t('aiTools.gpu.percent') + ' ( % )',
+            alignTicks: true,
-                position: 'right',
+        });
-                alignTicks: true,
+    }
-            },
+    chartsOption.value['loadPowerChart'] = {
-        ],
+        xData: baseDate,
+        yData,
+        yAxis,
         grid: isMobile.value ? { left: '15%', right: '15%', bottom: '20%' } : null,
         tooltip: {
             trigger: 'axis',
@@ -363,7 +367,7 @@ function initPowerCharts(baseDate: any, data: any) {
                 return res;
             },
         },
-        formatStr: '%',
+        formatStr: currentHide.value?.powerLimit ? 'W' : '%',
     };
 }
 

frontend/src/views/home/status/index.vue

@@ -513,7 +513,9 @@ const acceptParams = (current: Dashboard.CurrentInfo, base: Dashboard.BaseInfo):
         currentInfo.value.gpuData = currentInfo.value.gpuData || [];
         for (let i = 0; i < currentInfo.value.gpuData.length; i++) {
             chartsOption.value['gpu' + i] = {
-                title: 'GPU-' + currentInfo.value.gpuData[i].index,
+                title:
+                    (currentInfo.value.gpuData[i].type === 'ascend' ? 'NPU-' : 'GPU-') +
+                    currentInfo.value.gpuData[i].index,
                 data: formatNumber(Number(currentInfo.value.gpuData[i].gpuUtil.replaceAll(' %', ''))),
             };
         }