fanhuadesenlinnn/1Panel
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/1Panel-dev/1Panel.git synced 2026-09-20 08:03:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support Ascend 910B GPU monitoring (#13579)

Browse Source 
* feat: support Ascend 910B GPU monitoring

* chore: remove GPU test files
This commit is contained in:
ssongliu
2026-08-17 17:38:43 +08:00
committed by GitHub
parent b306bfa77a
commit 17a8835d59
13 changed files with 337 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agent/app/dto/dashboard.go
View File

@@ -156,6 +156,7 @@ type DiskInfo struct {
}
type GPUInfo struct {
Type string `json:"type"`
Index uint `json:"index"`
ProductName string `json:"productName"`
GPUUtil string `json:"gpuUtil"`

1
agent/app/dto/monitor.go
View File

@@ -49,6 +49,7 @@ type GPUChartHide struct {
GPU bool `json:"gpu"`
Memory bool `json:"memory"`
Power bool `json:"power"`
PowerLimit bool `json:"powerLimit"`
Temperature bool `json:"temperature"`
Speed bool `json:"speed"`
}

3
agent/app/repo/monitor.go
View File

@@ -73,6 +73,9 @@ func (u *MonitorRepo) CreateMonitorBase(model model.MonitorBase) error {
return global.MonitorDB.Create(&model).Error
}
func (s *MonitorRepo) BatchCreateMonitorGPU(list []model.MonitorGPU) error {
if len(list) == 0 {
return nil
}
return global.GPUMonitorDB.CreateInBatches(&list, len(list)).Error
}
func (u *MonitorRepo) BatchCreateMonitorIO(ioList []model.MonitorIO) error {

5
agent/app/service/monitor.go
View File

@@ -155,6 +155,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
if (item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A") && (item.PowerDraw == "" || item.PowerDraw == "N/A") {
chartHide.Power = true
}
chartHide.PowerLimit = item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A"
chartHide.Temperature = item.Temperature == "" || item.Temperature == "N/A"
chartHide.Speed = item.FanSpeed == "" || item.FanSpeed == "N/A"
data.ChartHide = append(data.ChartHide, chartHide)
@@ -174,6 +175,7 @@ func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
var chartHide dto.GPUChartHide
chartHide.GPU = true
chartHide.Speed = true
chartHide.PowerLimit = true
chartHide.ProductName = fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName)
if (item.Stats.MemoryUsed == "" || item.Stats.MemoryUsed == "N/A") && (item.Basic.Memory == "" || item.Basic.FreeMemory == "N/A") {
chartHide.Memory = true
@@ -344,6 +346,7 @@ func (m *MonitorService) Run() {
_ = monitorRepo.DelMonitorBase(timeForDelete)
_ = monitorRepo.DelMonitorIO(timeForDelete)
_ = monitorRepo.DelMonitorNet(timeForDelete)
_ = monitorRepo.DelMonitorGPU(timeForDelete)
}
func (m *MonitorService) loadDiskIO() {
@@ -599,6 +602,7 @@ func saveGPUDataToDB() {
}
gpuInfo, err := client.LoadGpuInfo()
if err != nil {
global.LOG.Errorf("load gpu monitor data failed, err: %v", err)
return
}
var list []model.MonitorGPU
@@ -631,6 +635,7 @@ func saveXPUDataToDB() {
}
xpuInfo, err := client.LoadGpuInfo()
if err != nil {
global.LOG.Errorf("load xpu monitor data failed, err: %v", err)
return
}
var list []model.MonitorGPU

164
agent/utils/ai_tools/gpu/ascend.go Normal file
View File

@@ -0,0 +1,164 @@
package gpu
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
)
var (
ascendVersionPattern = regexp.MustCompile(`(?i)\bVersion:\s*([^\s|]+)`)
ascendMemoryPattern = regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)`)
)
type AscendSMI struct{}
func (a AscendSMI) LoadGpuInfo() (*common.GpuInfo, error) {
cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second))
itemData, err := cmdMgr.RunWithStdout("npu-smi", "info")
if err != nil {
return nil, fmt.Errorf("calling npu-smi failed, %v", err)
}
return parseAscendSMI(itemData), nil
}
func parseAscendSMI(data string) *common.GpuInfo {
info := &common.GpuInfo{Type: "ascend"}
if match := ascendVersionPattern.FindStringSubmatch(data); len(match) == 2 {
info.DriverVersion = match[1]
}
processSection := false
deviceIndexes := make(map[uint]int)
var pending *common.GPU
for _, line := range strings.Split(data, "\n") {
if strings.Contains(line, "Process id") && strings.Contains(line, "Process memory") {
processSection = true
pending = nil
continue
}
cells := ascendTableCells(line)
if processSection {
if len(cells) != 4 && len(cells) != 5 {
continue
}
deviceFields := strings.Fields(cells[0])
if len(deviceFields) == 0 {
continue
}
deviceID, err := strconv.ParseUint(deviceFields[0], 10, 64)
if err != nil {
continue
}
index, ok := deviceIndexes[uint(deviceID)]
if !ok {
continue
}
processOffset := len(cells) - 3
info.GPUs[index].Processes = append(info.GPUs[index].Processes, common.Process{
Pid: strings.TrimSpace(cells[processOffset]),
Type: "NPU",
ProcessName: strings.TrimSpace(cells[processOffset+1]),
UsedMemory: ascendValueWithUnit(cells[processOffset+2], "MB"),
})
continue
}
if len(cells) != 3 {
continue
}
if pending == nil {
fields := strings.Fields(cells[0])
if len(fields) < 2 {
continue
}
deviceID, err := strconv.ParseUint(fields[0], 10, 64)
if err != nil {
continue
}
metrics := strings.Fields(cells[2])
if len(metrics) < 2 {
continue
}
pending = &common.GPU{
Type: "ascend",
Index: uint(deviceID),
ProductName: strings.Join(fields[1:], " "),
PersistenceMode: "N/A",
DisplayActive: "N/A",
ECC: "N/A",
FanSpeed: "N/A",
Temperature: ascendValueWithUnit(metrics[1], "C"),
PerformanceState: strings.TrimSpace(cells[1]),
PowerDraw: ascendValueWithUnit(metrics[0], "W"),
MaxPowerLimit: "N/A",
ComputeMode: "N/A",
MigMode: "N/A",
}
continue
}
metrics := strings.Fields(cells[2])
if len(metrics) == 0 {
pending = nil
continue
}
pending.BusID = strings.TrimSpace(cells[1])
pending.GPUUtil = ascendValueWithUnit(metrics[0], "%")
pending.MemUsed, pending.MemTotal = ascendMemoryUsage(cells[2])
deviceIndexes[pending.Index] = len(info.GPUs)
info.GPUs = append(info.GPUs, *pending)
pending = nil
}
return info
}
func ascendTableCells(line string) []string {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") {
return nil
}
parts := strings.Split(line, "|")
if len(parts) < 3 {
return nil
}
cells := make([]string, 0, len(parts)-2)
for _, part := range parts[1 : len(parts)-1] {
cells = append(cells, strings.TrimSpace(part))
}
return cells
}
func ascendMemoryUsage(value string) (string, string) {
matches := ascendMemoryPattern.FindAllStringSubmatch(value, -1)
if len(matches) == 0 {
return "N/A", "N/A"
}
// 910B exposes both generic memory and HBM. Prefer the last memory pool
// with a non-zero capacity, which selects HBM while retaining compatibility
// with devices that only expose Memory-Usage.
selected := matches[len(matches)-1]
for i := len(matches) - 1; i >= 0; i-- {
if total, err := strconv.ParseFloat(matches[i][2], 64); err == nil && total > 0 {
selected = matches[i]
break
}
}
return selected[1] + " MB", selected[2] + " MB"
}
func ascendValueWithUnit(value, unit string) string {
value = strings.TrimSpace(value)
if value == "" || strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") {
return "N/A"
}
return value + " " + unit
}

1
agent/utils/ai_tools/gpu/common/gpu_info.go
View File

@@ -9,6 +9,7 @@ type GpuInfo struct {
}
type GPU struct {
Type string `json:"type"`
Index uint `json:"index"`
ProductName string `json:"productName"`
PersistenceMode string `json:"persistenceMode"`

92
agent/utils/ai_tools/gpu/gpu.go
View File

@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/global"
@@ -18,8 +19,95 @@ import (
type NvidiaSMI struct{}
func New() (bool, NvidiaSMI) {
return cmd.Which("nvidia-smi"), NvidiaSMI{}
type SMI interface {
LoadGpuInfo() (*common.GpuInfo, error)
}
func New() (bool, SMI) {
var clients []SMI
if cmd.Which("nvidia-smi") {
clients = append(clients, NvidiaSMI{})
}
if cmd.Which("npu-smi") {
clients = append(clients, AscendSMI{})
}
if len(clients) == 0 {
return false, nil
}
if len(clients) == 1 {
return true, clients[0]
}
return true, multiSMI{clients: clients}
}
type multiSMI struct {
clients []SMI
}
type smiResult struct {
info *common.GpuInfo
err error
}
func (m multiSMI) LoadGpuInfo() (*common.GpuInfo, error) {
results := make([]smiResult, len(m.clients))
var wg sync.WaitGroup
for index, client := range m.clients {
wg.Add(1)
go func() {
defer wg.Done()
results[index].info, results[index].err = client.LoadGpuInfo()
}()
}
wg.Wait()
merged := &common.GpuInfo{}
var (
errs []error
types []string
driverVersions []string
)
for _, result := range results {
if result.err != nil {
errs = append(errs, result.err)
continue
}
if result.info == nil {
continue
}
deviceType := result.info.Type
if deviceType != "" {
types = append(types, deviceType)
}
if result.info.DriverVersion != "" {
driverVersions = append(driverVersions, fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), result.info.DriverVersion))
}
if result.info.CudaVersion != "" {
merged.CudaVersion = result.info.CudaVersion
}
for _, device := range result.info.GPUs {
if device.Type == "" {
device.Type = deviceType
}
merged.GPUs = append(merged.GPUs, device)
}
}
if len(types) == 1 {
merged.Type = types[0]
} else if len(types) > 1 {
merged.Type = "mixed"
}
if len(driverVersions) == 1 {
parts := strings.SplitN(driverVersions[0], ": ", 2)
merged.DriverVersion = parts[len(parts)-1]
} else {
merged.DriverVersion = strings.Join(driverVersions, "；")
}
if len(merged.GPUs) == 0 && len(errs) > 0 {
return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...))
}
return merged, nil
}
func (n NvidiaSMI) LoadGpuInfo() (*common.GpuInfo, error) {

1
agent/utils/ai_tools/gpu/schema/parser.go
View File

@@ -23,6 +23,7 @@ func Parse(buf []byte, version string) (*common.GpuInfo, error) {
}
for i := 0; i < len(s.Gpu); i++ {
var gpuItem common.GPU
gpuItem.Type = "nvidia"
gpuItem.Index = uint(i)
gpuItem.ProductName = s.Gpu[i].ProductName
gpuItem.PersistenceMode = s.Gpu[i].PersistenceMode

2
frontend/src/api/interface/ai.ts
View File

@@ -28,6 +28,7 @@ export namespace AI {
gpu: GPU[];
}
export interface GPU {
type: string;
index: number;
productName: string;
persistenceMode: string;
@@ -63,6 +64,7 @@ export namespace AI {
gpu: boolean;
memory: boolean;
power: boolean;
powerLimit: boolean;
temperature: boolean;
speed: boolean;
}

1
frontend/src/api/interface/dashboard.ts
View File

@@ -144,6 +144,7 @@ export namespace Dashboard {
inodesUsedPercent: number;
}
export interface GPUInfo {
type: string;
index: number;
productName: string;
gpuUtil: string;

55
frontend/src/views/ai/gpu/current/index.vue
View File

@@ -1,7 +1,7 @@
<template>
<div>
<RouterMenu />
<div v-if="gpuType == 'nvidia'">
<div v-if="gpuType !== 'xpu'">
<LayoutContent
v-loading="loading"
:title="$t('aiTools.gpu.gpu')"
@@ -14,15 +14,27 @@
</template>
<template #main>
<el-descriptions direction="vertical" :column="14" border>
<el-descriptions-item :label="$t('aiTools.gpu.driverVersion')" width="50%" :span="7">
<el-descriptions-item
:label="$t('aiTools.gpu.driverVersion')"
width="50%"
:span="gpuInfo.cudaVersion ? 7 : 14"
>
{{ gpuInfo.driverVersion }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.cudaVersion')" :span="7">
<el-descriptions-item
v-if="gpuInfo.cudaVersion"
:label="$t('aiTools.gpu.cudaVersion')"
:span="7"
>
{{ gpuInfo.cudaVersion }}
</el-descriptions-item>
</el-descriptions>
<el-collapse v-model="activeNames" class="card-interval">
<el-collapse-item v-for="item in gpuInfo.gpu" :key="item.index" :name="item.index">
<el-collapse-item
v-for="item in gpuInfo.gpu"
:key="item.type + '-' + item.index"
:name="item.type + '-' + item.index"
>
<template #title>
<span class="name-class">{{ item.index + '. ' + item.productName }}</span>
</template>
@@ -45,8 +57,13 @@
<el-descriptions-item>
<template #label>
<div class="cell-item">
{{ $t('aiTools.gpu.performanceState') }}
{{
item.type === 'ascend'
? $t('commons.table.status')
: $t('aiTools.gpu.performanceState')
}}
<el-tooltip
v-if="item.type === 'nvidia'"
placement="top"
:content="$t('aiTools.gpu.performanceStateHelper')"
>
@@ -57,19 +74,20 @@
{{ item.performanceState }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.powerUsage')">
{{ item.powerDraw }} / {{ item.maxPowerLimit }}
{{ item.powerDraw }}
<template v-if="item.maxPowerLimit !== 'N/A'">/ {{ item.maxPowerLimit }}</template>
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.memoryUsage')">
{{ item.memUsed }} / {{ item.memTotal }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.fanSpeed')">
<el-descriptions-item v-if="item.type === 'nvidia'" :label="$t('aiTools.gpu.fanSpeed')">
{{ item.fanSpeed }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.busID')">
{{ item.busID }}
</el-descriptions-item>
<el-descriptions-item>
<el-descriptions-item v-if="item.type === 'nvidia'">
<template #label>
<div class="cell-item">
{{ $t('aiTools.gpu.persistenceMode') }}
@@ -83,14 +101,17 @@
</template>
{{ $t('aiTools.gpu.' + item.persistenceMode.toLowerCase()) }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.displayActive')">
<el-descriptions-item
v-if="item.type === 'nvidia'"
:label="$t('aiTools.gpu.displayActive')"
>
{{
lowerCase(item.displayActive) === 'disabled'
? $t('aiTools.gpu.displayActiveF')
: $t('aiTools.gpu.displayActiveT')
}}
</el-descriptions-item>
<el-descriptions-item>
<el-descriptions-item v-if="item.type === 'nvidia'">
<template #label>
<div class="cell-item">
Uncorr. ECC
@@ -101,7 +122,10 @@
</template>
{{ loadEcc(item.ecc) }}
</el-descriptions-item>
<el-descriptions-item :label="$t('aiTools.gpu.computeMode')">
<el-descriptions-item
v-if="item.type === 'nvidia'"
:label="$t('aiTools.gpu.computeMode')"
>
<template #label>
<div class="cell-item">
{{ $t('aiTools.gpu.computeMode') }}
@@ -121,7 +145,7 @@
</template>
{{ loadComputeMode(item.computeMode) }}
</el-descriptions-item>
<el-descriptions-item label="MIG.M">
<el-descriptions-item v-if="item.type === 'nvidia'" label="MIG.M">
<template #label>
<div class="cell-item">
MIG M.
@@ -268,7 +292,7 @@ const search = async () => {
.then((res) => {
loading.value = false;
gpuType.value = res.data.type;
if (res.data.type == 'nvidia') {
if (res.data.type !== 'xpu') {
gpuInfo.value = res.data;
} else {
xpuInfo.value = res.data;
@@ -281,7 +305,12 @@ const search = async () => {
const refresh = async () => {
const res = await loadGPUInfo();
gpuType.value = res.data.type;
if (res.data.type !== 'xpu') {
gpuInfo.value = res.data;
} else {
xpuInfo.value = res.data;
}
};
const lowerCase = (val: string) => {

24
frontend/src/views/ai/gpu/history/index.vue
View File

@@ -319,13 +319,15 @@ function initMemoryCharts(baseDate: any, data: any) {
}
function initPowerCharts(baseDate: any, data: any) {
chartsOption.value['loadPowerChart'] = {
xData: baseDate,
yData: [
const yData: any[] = [
{
name: i18n.global.t('aiTools.gpu.powerCurrent'),
data: data.powerUsed,
},
];
const yAxis: any[] = [{ type: 'value', name: i18n.global.t('aiTools.gpu.power') }];
if (!currentHide.value?.powerLimit) {
yData.push(
{
name: i18n.global.t('aiTools.gpu.powerLimit'),
data: data.powerTotal,
@@ -335,16 +337,18 @@ function initPowerCharts(baseDate: any, data: any) {
data: data.powerPercent,
yAxisIndex: 1,
},
],
yAxis: [
{ type: 'value', name: i18n.global.t('aiTools.gpu.power') },
{
);
yAxis.push({
type: 'value',
name: i18n.global.t('aiTools.gpu.percent') + ' ( % )',
position: 'right',
alignTicks: true,
},
],
});
}
chartsOption.value['loadPowerChart'] = {
xData: baseDate,
yData,
yAxis,
grid: isMobile.value ? { left: '15%', right: '15%', bottom: '20%' } : null,
tooltip: {
trigger: 'axis',
@@ -363,7 +367,7 @@ function initPowerCharts(baseDate: any, data: any) {
return res;
},
},
formatStr: '%',
formatStr: currentHide.value?.powerLimit ? 'W' : '%',
};
}

4
frontend/src/views/home/status/index.vue
View File

@@ -513,7 +513,9 @@ const acceptParams = (current: Dashboard.CurrentInfo, base: Dashboard.BaseInfo):
currentInfo.value.gpuData = currentInfo.value.gpuData || [];
for (let i = 0; i < currentInfo.value.gpuData.length; i++) {
chartsOption.value['gpu' + i] = {
title: 'GPU-' + currentInfo.value.gpuData[i].index,
title:
(currentInfo.value.gpuData[i].type === 'ascend' ? 'NPU-' : 'GPU-') +
currentInfo.value.gpuData[i].index,
data: formatNumber(Number(currentInfo.value.gpuData[i].gpuUtil.replaceAll(' %', ''))),
};
}