diff --git a/agent/app/api/v2/gpu.go b/agent/app/api/v2/gpu.go index 11620401c..e9aedd740 100644 --- a/agent/app/api/v2/gpu.go +++ b/agent/app/api/v2/gpu.go @@ -3,9 +3,8 @@ package v2 import ( "github.com/1Panel-dev/1Panel/agent/app/api/v2/helper" "github.com/1Panel-dev/1Panel/agent/app/dto" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/accelerator" "github.com/gin-gonic/gin" ) @@ -17,27 +16,20 @@ import ( // @Security Timestamp // @Router /ai/gpu/load [get] func (b *BaseApi) LoadGpuInfo(c *gin.Context) { - ok, client := gpu.New() + ok, client := accelerator.New() if ok { - info, err := client.LoadGpuInfo() + snapshot, err := client.Collect(c.Request.Context()) if err != nil { helper.BadRequest(c, err) return } - helper.SuccessWithData(c, info) - return - } - xpuOK, xpuClient := xpu.New() - if xpuOK { - info, err := xpuClient.LoadGpuInfo() - if err != nil { - helper.BadRequest(c, err) - return + if warning := snapshot.Warning(); warning != nil { + global.LOG.Warnf("load realtime accelerator data partially failed, err: %v", warning) } - helper.SuccessWithData(c, info) + helper.SuccessWithData(c, &snapshot.Info) return } - helper.SuccessWithData(c, &common.GpuInfo{}) + helper.SuccessWithData(c, &accelerator.Info{}) } // @Tags AI diff --git a/agent/app/dto/dashboard.go b/agent/app/dto/dashboard.go index c7d83ac2e..0e36e3178 100644 --- a/agent/app/dto/dashboard.go +++ b/agent/app/dto/dashboard.go @@ -121,6 +121,7 @@ type DashboardCurrent struct { NetBytesRecv uint64 `json:"netBytesRecv"` GPUData []GPUInfo `json:"gpuData"` + NPUData []NPUInfo `json:"npuData"` XPUData []XPUInfo `json:"xpuData"` TopCPUItems []Process `json:"topCPUItems"` @@ -158,7 +159,10 @@ type DiskInfo struct { type GPUInfo struct { Type string `json:"type"` Index uint `json:"index"` + NPUIndex uint `json:"npuIndex"` + ChipIndex uint `json:"chipIndex"` ProductName string `json:"productName"` + BusID string `json:"busID"` GPUUtil string `json:"gpuUtil"` Temperature string `json:"temperature"` PerformanceState string `json:"performanceState"` @@ -171,6 +175,27 @@ type GPUInfo struct { FanSpeed string `json:"fanSpeed"` } +type NPUInfo struct { + Type string `json:"type"` + Index uint `json:"index"` + NPUIndex uint `json:"npuIndex"` + ChipIndex uint `json:"chipIndex"` + ProductName string `json:"productName"` + BusID string `json:"busID"` + Health string `json:"health"` + Temperature string `json:"temperature"` + PowerDraw string `json:"powerDraw"` + AICore string `json:"aiCore"` + MemUsed string `json:"memUsed"` + MemTotal string `json:"memTotal"` + MemoryUsed string `json:"memoryUsed"` + MemoryTotal string `json:"memoryTotal"` + HBMUsed string `json:"hbmUsed"` + HBMTotal string `json:"hbmTotal"` + HugepagesUsed string `json:"hugepagesUsed"` + HugepagesTotal string `json:"hugepagesTotal"` +} + type AppLauncher struct { Key string `json:"key"` Type string `json:"type"` @@ -203,11 +228,13 @@ type LauncherOption struct { } type XPUInfo struct { - DeviceID int `json:"deviceID"` - DeviceName string `json:"deviceName"` - Memory string `json:"memory"` - Temperature string `json:"temperature"` - MemoryUsed string `json:"memoryUsed"` - Power string `json:"power"` - MemoryUtil string `json:"memoryUtil"` + DeviceID int `json:"deviceID"` + DeviceName string `json:"deviceName"` + PciBdfAddress string `json:"pciBdfAddress"` + Memory string `json:"memory"` + Temperature string `json:"temperature"` + GPUUtil string `json:"gpuUtil"` + MemoryUsed string `json:"memoryUsed"` + Power string `json:"power"` + MemoryUtil string `json:"memoryUtil"` } diff --git a/agent/app/dto/monitor.go b/agent/app/dto/monitor.go index 6f229adb7..1056faf29 100644 --- a/agent/app/dto/monitor.go +++ b/agent/app/dto/monitor.go @@ -45,6 +45,7 @@ type MonitorGPUOptions struct { } type GPUChartHide struct { ProductName string `json:"productName"` + Type string `json:"type"` Process bool `json:"process"` GPU bool `json:"gpu"` Memory bool `json:"memory"` diff --git a/agent/app/service/dashboard.go b/agent/app/service/dashboard.go index 1ce2497e1..c41720e65 100644 --- a/agent/app/service/dashboard.go +++ b/agent/app/service/dashboard.go @@ -18,8 +18,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/accelerator" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/controller" @@ -244,8 +243,7 @@ func (u *DashboardService) LoadCurrentInfo(ioOption string, netOption string) *d currentInfo.SwapMemoryUsedPercent = swapInfo.UsedPercent currentInfo.DiskData = loadDiskInfo() - currentInfo.GPUData = loadGPUInfo() - currentInfo.XPUData = loadXpuInfo() + currentInfo.GPUData, currentInfo.NPUData, currentInfo.XPUData = loadAcceleratorInfo() if ioOption == "all" { diskInfo, _ := disk.IOCounters() @@ -569,32 +567,64 @@ func loadDiskInfo() []dto.DiskInfo { return datas } -func loadGPUInfo() []dto.GPUInfo { - ok, client := gpu.New() - var list []interface{} - if ok { - info, err := client.LoadGpuInfo() - if err != nil || len(info.GPUs) == 0 { - return nil - } - for _, item := range info.GPUs { - list = append(list, item) +func loadAcceleratorInfo() ([]dto.GPUInfo, []dto.NPUInfo, []dto.XPUInfo) { + ok, client := accelerator.New() + if !ok { + return nil, nil, nil + } + snapshot, err := client.Collect(context.Background()) + if err != nil || len(snapshot.Devices) == 0 { + return nil, nil, nil + } + if warning := snapshot.Warning(); warning != nil { + global.LOG.Warnf("load accelerator dashboard data partially failed, err: %v", warning) + } + + var ( + gpuData []dto.GPUInfo + npuData []dto.NPUInfo + xpuData []dto.XPUInfo + ) + for _, device := range snapshot.Devices { + switch device.Kind { + case accelerator.KindGPU: + if device.GPU == nil { + continue + } + var dataItem dto.GPUInfo + if err := copier.Copy(&dataItem, device.GPU); err != nil { + continue + } + dataItem.PowerUsage = dataItem.PowerDraw + " / " + dataItem.MaxPowerLimit + dataItem.MemoryUsage = dataItem.MemUsed + " / " + dataItem.MemTotal + gpuData = append(gpuData, dataItem) + case accelerator.KindNPU: + if device.NPU == nil { + continue + } + var dataItem dto.NPUInfo + if err := copier.Copy(&dataItem, device.NPU); err != nil { + continue + } + npuData = append(npuData, dataItem) + case accelerator.KindXPU: + if device.XPU == nil { + continue + } + xpuData = append(xpuData, dto.XPUInfo{ + DeviceID: device.Index, + DeviceName: device.Name, + PciBdfAddress: device.BusID, + Memory: device.XPU.Basic.Memory, + Temperature: device.Metrics.Temperature.Display, + GPUUtil: device.Metrics.Utilization.Display, + MemoryUsed: device.Metrics.MemoryUsed.Display, + Power: device.Metrics.Power.Display, + MemoryUtil: device.Metrics.MemoryUtil.Display, + }) } } - if len(list) == 0 { - return nil - } - var data []dto.GPUInfo - for _, gpu := range list { - var dataItem dto.GPUInfo - if err := copier.Copy(&dataItem, &gpu); err != nil { - continue - } - dataItem.PowerUsage = dataItem.PowerDraw + " / " + dataItem.MaxPowerLimit - dataItem.MemoryUsage = dataItem.MemUsed + " / " + dataItem.MemTotal - data = append(data, dataItem) - } - return data + return gpuData, npuData, xpuData } type AppLauncher struct { @@ -610,32 +640,6 @@ func ArryContains(arr []string, element string) bool { return false } -func loadXpuInfo() []dto.XPUInfo { - var list []interface{} - ok, xpuClient := xpu.New() - if ok { - xpus, err := xpuClient.LoadDashData() - if err != nil || len(xpus) == 0 { - return nil - } - for _, item := range xpus { - list = append(list, item) - } - } - if len(list) == 0 { - return nil - } - var data []dto.XPUInfo - for _, gpu := range list { - var dataItem dto.XPUInfo - if err := copier.Copy(&dataItem, &gpu); err != nil { - continue - } - data = append(data, dataItem) - } - return data -} - func loadOutboundIP() string { conn, err := network.Dial("udp", "8.8.8.8:80") diff --git a/agent/app/service/monitor.go b/agent/app/service/monitor.go index 98bd166f8..1f4ffe66e 100644 --- a/agent/app/service/monitor.go +++ b/agent/app/service/monitor.go @@ -8,7 +8,6 @@ import ( "path/filepath" "sort" "strconv" - "strings" "sync" "time" @@ -19,8 +18,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/global" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/accelerator" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/psutil" "github.com/robfig/cron/v3" @@ -130,65 +128,75 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions { var data dto.MonitorGPUOptions - gpuExist, gpuClient := gpu.New() - xpuExist, xpuClient := xpu.New() - if !gpuExist && !xpuExist { + exist, client := accelerator.New() + if !exist { return data } - if gpuExist { - data.GPUType = "gpu" - gpuInfo, err := gpuClient.LoadGpuInfo() - if err != nil || len(gpuInfo.GPUs) == 0 { - global.LOG.Error("Load GPU info failed or no GPU found, err: ", err) - return data - } - sort.Slice(gpuInfo.GPUs, func(i, j int) bool { - return gpuInfo.GPUs[i].Index < gpuInfo.GPUs[j].Index - }) - for _, item := range gpuInfo.GPUs { - var chartHide dto.GPUChartHide - chartHide.ProductName = fmt.Sprintf("%d - %s", item.Index, item.ProductName) - chartHide.GPU = item.GPUUtil == "" || item.GPUUtil == "N/A" - if (item.MemTotal == "" || item.MemTotal == "N/A") && (item.MemUsed == "" || item.MemUsed == "N/A") { - chartHide.Memory = true - } - if (item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A") && (item.PowerDraw == "" || item.PowerDraw == "N/A") { - chartHide.Power = true - } - chartHide.PowerLimit = item.MaxPowerLimit == "" || item.MaxPowerLimit == "N/A" - chartHide.Temperature = item.Temperature == "" || item.Temperature == "N/A" - chartHide.Speed = item.FanSpeed == "" || item.FanSpeed == "N/A" - data.ChartHide = append(data.ChartHide, chartHide) - data.Options = append(data.Options, fmt.Sprintf("%d - %s", item.Index, item.ProductName)) - } + snapshot, err := client.Collect(context.Background()) + if err != nil { + global.LOG.Errorf("Load accelerator info failed, err: %v", err) return data - } else { + } + if warning := snapshot.Warning(); warning != nil { + global.LOG.Warnf("Load accelerator info partially failed, err: %v", warning) + } + return loadGPUOptions(snapshot) +} + +func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions { + var data dto.MonitorGPUOptions + hasGPUOrNPU := false + hasXPU := false + for _, item := range snapshot.Devices { + if item.Kind == accelerator.KindXPU { + hasXPU = true + } else { + hasGPUOrNPU = true + } + } + switch { + case hasGPUOrNPU && hasXPU: + data.GPUType = "mixed" + case hasXPU: data.GPUType = "xpu" - xpu, err := xpuClient.LoadGpuInfo() - if err != nil || len(xpu.Xpu) == 0 { - global.LOG.Error("Load XPU info failed or no XPU found, err: ", err) - } - sort.Slice(xpu.Xpu, func(i, j int) bool { - return xpu.Xpu[i].Basic.DeviceID < xpu.Xpu[j].Basic.DeviceID - }) - for _, item := range xpu.Xpu { - var chartHide dto.GPUChartHide - chartHide.GPU = true - chartHide.Speed = true - chartHide.PowerLimit = true - chartHide.ProductName = fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName) - if (item.Stats.MemoryUsed == "" || item.Stats.MemoryUsed == "N/A") && (item.Basic.Memory == "" || item.Basic.FreeMemory == "N/A") { - chartHide.Memory = true - } - if item.Stats.Power == "" || item.Stats.Power == "N/A" { - chartHide.Power = true - } - chartHide.Temperature = item.Stats.Temperature == "" || item.Stats.Temperature == "N/A" - data.ChartHide = append(data.ChartHide, chartHide) - data.Options = append(data.Options, fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName)) - } - return data + case hasGPUOrNPU: + data.GPUType = "gpu" } + + sort.Slice(snapshot.Devices, func(i, j int) bool { + if snapshot.Devices[i].Kind != snapshot.Devices[j].Kind { + return snapshot.Devices[i].Kind < snapshot.Devices[j].Kind + } + if snapshot.Devices[i].Vendor != snapshot.Devices[j].Vendor { + return snapshot.Devices[i].Vendor < snapshot.Devices[j].Vendor + } + if snapshot.Devices[i].NPUIndex != snapshot.Devices[j].NPUIndex { + return snapshot.Devices[i].NPUIndex < snapshot.Devices[j].NPUIndex + } + if snapshot.Devices[i].ChipIndex != snapshot.Devices[j].ChipIndex { + return snapshot.Devices[i].ChipIndex < snapshot.Devices[j].ChipIndex + } + return snapshot.Devices[i].Index < snapshot.Devices[j].Index + }) + for _, item := range snapshot.Devices { + optionType := "gpu" + if item.Kind == accelerator.KindXPU { + optionType = "xpu" + } + chartHide := dto.GPUChartHide{ + ProductName: item.Label, + Type: optionType, + GPU: !item.Capabilities.Utilization, + Memory: !item.Capabilities.Memory, + Power: !item.Capabilities.Power, + PowerLimit: !item.Capabilities.PowerLimit, + Temperature: !item.Capabilities.Temperature, + Speed: !item.Capabilities.FanSpeed, + } + data.ChartHide = append(data.ChartHide, chartHide) + data.Options = append(data.Options, chartHide.ProductName) + } + return data } func (m *MonitorService) LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.MonitorGPUData, error) { @@ -299,8 +307,7 @@ func (m *MonitorService) CleanData() error { } func (m *MonitorService) Run() { - saveGPUDataToDB() - saveXPUDataToDB() + saveAcceleratorDataToDB() var itemModel model.MonitorBase totalPercent, _ := cpu.Percent(3*time.Second, false) if len(totalPercent) == 1 { @@ -595,95 +602,56 @@ func StartMonitor(removeBefore bool, interval string) error { return nil } -func saveGPUDataToDB() { - exist, client := gpu.New() +func saveAcceleratorDataToDB() { + exist, client := accelerator.New() if !exist { return } - gpuInfo, err := client.LoadGpuInfo() + snapshot, err := client.Collect(context.Background()) if err != nil { - global.LOG.Errorf("load gpu monitor data failed, err: %v", err) + global.LOG.Errorf("load accelerator monitor data failed, err: %v", err) return } - var list []model.MonitorGPU - for _, gpuItem := range gpuInfo.GPUs { - item := model.MonitorGPU{ - ProductName: fmt.Sprintf("%d - %s", gpuItem.Index, gpuItem.ProductName), - GPUUtil: loadGPUInfoFloat(gpuItem.GPUUtil), - Temperature: loadGPUInfoFloat(gpuItem.Temperature), - PowerDraw: loadGPUInfoFloat(gpuItem.PowerDraw), - MaxPowerLimit: loadGPUInfoFloat(gpuItem.MaxPowerLimit), - MemUsed: loadGPUInfoFloat(gpuItem.MemUsed), - MemTotal: loadGPUInfoFloat(gpuItem.MemTotal), - FanSpeed: loadGPUInfoInt(gpuItem.FanSpeed), - } - process, _ := json.Marshal(gpuItem.Processes) - if len(process) != 0 { - item.Processes = string(process) - } - list = append(list, item) + if warning := snapshot.Warning(); warning != nil { + global.LOG.Warnf("load accelerator monitor data partially failed, err: %v", warning) + } + list := make([]model.MonitorGPU, 0, len(snapshot.Devices)) + for _, device := range snapshot.Devices { + list = append(list, newMonitorGPU(device)) } if err := repo.NewIMonitorRepo().BatchCreateMonitorGPU(list); err != nil { - global.LOG.Errorf("batch create gpu monitor data failed, err: %v", err) - return + global.LOG.Errorf("batch create accelerator monitor data failed, err: %v", err) } } -func saveXPUDataToDB() { - exist, client := xpu.New() - if !exist { - return + +func newMonitorGPU(device accelerator.Device) model.MonitorGPU { + item := model.MonitorGPU{ + ProductName: device.Label, + GPUUtil: device.Metrics.Utilization.ValueOrZero(), + Temperature: device.Metrics.Temperature.ValueOrZero(), + PowerDraw: device.Metrics.Power.ValueOrZero(), + MaxPowerLimit: device.Metrics.PowerLimit.ValueOrZero(), + MemUsed: device.Metrics.MemoryUsed.ValueOrZero(), + MemTotal: device.Metrics.MemoryTotal.ValueOrZero(), + FanSpeed: int(device.Metrics.FanSpeed.ValueOrZero()), } - xpuInfo, err := client.LoadGpuInfo() - if err != nil { - global.LOG.Errorf("load xpu monitor data failed, err: %v", err) - return + if len(device.Processes) == 0 { + return item } - var list []model.MonitorGPU - for _, xpuItem := range xpuInfo.Xpu { - item := model.MonitorGPU{ - ProductName: fmt.Sprintf("%d - %s", xpuItem.Basic.DeviceID, xpuItem.Basic.DeviceName), - Temperature: loadGPUInfoFloat(xpuItem.Stats.Temperature), - PowerDraw: loadGPUInfoFloat(xpuItem.Stats.Power), - MemUsed: loadGPUInfoFloat(xpuItem.Stats.MemoryUsed), - MemTotal: loadGPUInfoFloat(xpuItem.Basic.Memory), - } - if len(xpuItem.Processes) != 0 { - var processItem []dto.GPUProcess - for _, ps := range xpuItem.Processes { - processItem = append(processItem, dto.GPUProcess{ - Pid: fmt.Sprintf("%v", ps.PID), - Type: ps.SHR, - ProcessName: ps.Command, - UsedMemory: ps.Memory, - }) - } - process, _ := json.Marshal(processItem) - if len(process) != 0 { - item.Processes = string(process) - } - } - list = append(list, item) + processes := make([]dto.GPUProcess, 0, len(device.Processes)) + for _, process := range device.Processes { + processes = append(processes, dto.GPUProcess{ + Pid: process.PID, + Type: process.Type, + ProcessName: process.Name, + UsedMemory: process.Memory, + }) } - if err := repo.NewIMonitorRepo().BatchCreateMonitorGPU(list); err != nil { - global.LOG.Errorf("batch create gpu monitor data failed, err: %v", err) - return + processData, err := json.Marshal(processes) + if err == nil { + item.Processes = string(processData) } -} -func loadGPUInfoInt(val string) int { - val = strings.TrimSuffix(val, "%") - val = strings.TrimSpace(val) - data, _ := strconv.Atoi(val) - return data -} -func loadGPUInfoFloat(val string) float64 { - val = strings.TrimSpace(val) - suffixes := []string{"W", "MB", "MiB", "°C", "C", "%"} - for _, suffix := range suffixes { - val = strings.TrimSuffix(val, suffix) - } - val = strings.TrimSpace(val) - data, _ := strconv.ParseFloat(val, 64) - return data + return item } func sumDiskIOCounters(ioStats map[string]disk.IOCountersStat) disk.IOCountersStat { diff --git a/agent/utils/ai_tools/accelerator/accelerator.go b/agent/utils/ai_tools/accelerator/accelerator.go new file mode 100644 index 000000000..67f929c51 --- /dev/null +++ b/agent/utils/ai_tools/accelerator/accelerator.go @@ -0,0 +1,134 @@ +package accelerator + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" +) + +type Client struct { + providers []Provider +} + +type providerResult struct { + snapshot *ProviderSnapshot + err error +} + +func New() (bool, Client) { + return newClient() +} + +func NewAll() (bool, Client) { + return New() +} + +func newClient() (bool, Client) { + client := Client{} + if available, gpuClient := gpu.New(); available { + client.providers = append(client.providers, gpuProvider{client: gpuClient}) + } + if available, npuClient := npu.New(); available { + client.providers = append(client.providers, npuProvider{client: npuClient}) + } + if available, xpuClient := xpu.New(); available { + client.providers = append(client.providers, xpuProvider{client: xpuClient}) + } + return len(client.providers) > 0, client +} + +func (c Client) LoadInfo() (*Info, error) { + snapshot, err := c.Collect(context.Background()) + if err != nil { + return nil, err + } + return &snapshot.Info, nil +} + +func (c Client) Collect(ctx context.Context) (*Snapshot, error) { + results := make([]providerResult, len(c.providers)) + var wg sync.WaitGroup + for index, item := range c.providers { + wg.Add(1) + go func() { + defer wg.Done() + providerSnapshot, err := item.Collect(ctx) + if err != nil { + err = fmt.Errorf("%s provider failed: %w", item.Kind(), err) + } + results[index] = providerResult{snapshot: providerSnapshot, err: err} + }() + } + wg.Wait() + + snapshot := &Snapshot{DriverVersions: make(map[string]string)} + var ( + errs []error + active []*ProviderSnapshot + xpuVersion string + ) + for _, result := range results { + if result.err != nil { + errs = append(errs, result.err) + continue + } + if result.snapshot == nil || len(result.snapshot.Devices) == 0 { + continue + } + item := result.snapshot + active = append(active, item) + snapshot.Devices = append(snapshot.Devices, item.Devices...) + snapshot.Info.GPUs = append(snapshot.Info.GPUs, item.GPUs...) + snapshot.Info.NPUs = append(snapshot.Info.NPUs, item.NPUs...) + snapshot.Info.XPUs = append(snapshot.Info.XPUs, item.XPUs...) + if item.CudaVersion != "" { + snapshot.Info.CudaVersion = item.CudaVersion + } + if item.DriverVersion != "" { + snapshot.DriverVersions[item.Type] = item.DriverVersion + if item.Type == "xpu" { + xpuVersion = item.DriverVersion + } + } + } + snapshot.Warnings = append(snapshot.Warnings, errs...) + if len(snapshot.Devices) == 0 && len(errs) > 0 { + return nil, fmt.Errorf("calling accelerator monitoring tools failed: %w", errors.Join(errs...)) + } + + snapshot.Info.XPUDriverVersion = xpuVersion + snapshot.Info.Type, snapshot.Info.DriverVersion = mergeProviderMetadata(active) + return snapshot, nil +} + +func mergeProviderMetadata(items []*ProviderSnapshot) (string, string) { + if len(items) == 0 { + return "", "" + } + resultType := items[0].Type + if len(items) > 1 || resultType == "mixed" { + resultType = "mixed" + } + driverVersions := make([]string, 0, len(items)) + for _, item := range items { + if item.DriverVersion == "" { + continue + } + if len(items) == 1 && item.Type != "mixed" { + driverVersions = append(driverVersions, item.DriverVersion) + continue + } + if item.Type == "mixed" { + driverVersions = append(driverVersions, item.DriverVersion) + } else { + driverVersions = append(driverVersions, fmt.Sprintf("%s: %s", strings.ToUpper(item.Type), item.DriverVersion)) + } + } + return resultType, strings.Join(driverVersions, "；") +} diff --git a/agent/utils/ai_tools/accelerator/metric.go b/agent/utils/ai_tools/accelerator/metric.go new file mode 100644 index 000000000..4234c0931 --- /dev/null +++ b/agent/utils/ai_tools/accelerator/metric.go @@ -0,0 +1,112 @@ +package accelerator + +import ( + "strconv" + "strings" + + "github.com/1Panel-dev/1Panel/agent/utils/re" +) + +func metric(display, unit string) Metric { + result := Metric{Display: display, Unit: unit} + value := strings.TrimSpace(display) + if value == "" || strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") { + return result + } + matched := re.GetRegex(re.AcceleratorMetricValuePattern).FindStringSubmatch(value) + if len(matched) != 3 { + return result + } + parsed, err := strconv.ParseFloat(matched[1], 64) + if err != nil { + return result + } + normalized, ok := convertMetricUnit(parsed, matched[2], unit) + if !ok { + return result + } + result.Value = &normalized + return result +} + +func memoryMetric(display string) Metric { + return metric(display, "MiB") +} + +func convertMetricUnit(value float64, sourceUnit, targetUnit string) (float64, bool) { + source := normalizeUnit(sourceUnit) + target := normalizeUnit(targetUnit) + if source == "" { + source = target + } + + switch target { + case "%": + return value, source == "%" || source == "percent" || source == "pct" + case "w": + switch source { + case "w", "watt", "watts": + return value, true + case "mw": + return value / 1000, true + case "kw": + return value * 1000, true + } + case "°c": + switch source { + case "c", "°c", "℃": + return value, true + case "f", "°f", "℉": + return (value - 32) * 5 / 9, true + case "k": + return value - 273.15, true + } + case "mhz": + switch source { + case "hz": + return value / 1_000_000, true + case "khz": + return value / 1000, true + case "mhz": + return value, true + case "ghz": + return value * 1000, true + } + case "mib": + switch source { + case "b", "byte", "bytes": + return value / (1024 * 1024), true + case "kb": + return value * 1000 / (1024 * 1024), true + case "kib": + return value / 1024, true + case "mb": + return value * 1_000_000 / (1024 * 1024), true + case "mib": + return value, true + case "gb": + return value * 1_000_000_000 / (1024 * 1024), true + case "gib": + return value * 1024, true + case "tb": + return value * 1_000_000_000_000 / (1024 * 1024), true + case "tib": + return value * 1024 * 1024, true + } + default: + return value, source == target + } + return 0, false +} + +func normalizeUnit(unit string) string { + return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(unit)), "")) +} + +func normalizedMemoryDisplay(display string) string { + value := memoryMetric(display) + if !value.Available() { + return display + } + return strconv.FormatFloat(value.ValueOrZero(), 'f', -1, 64) + " MiB" +} diff --git a/agent/utils/ai_tools/accelerator/normalize.go b/agent/utils/ai_tools/accelerator/normalize.go new file mode 100644 index 000000000..e1ca60542 --- /dev/null +++ b/agent/utils/ai_tools/accelerator/normalize.go @@ -0,0 +1,130 @@ +package accelerator + +import ( + "fmt" + "strconv" + "strings" + + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" +) + +func normalizeGPU(item *gpu.Device) Device { + metrics := Metrics{ + Utilization: metric(item.GPUUtil, "%"), + Temperature: metric(item.Temperature, "°C"), + Power: metric(item.PowerDraw, "W"), + PowerLimit: metric(item.MaxPowerLimit, "W"), + MemoryUsed: memoryMetric(item.MemUsed), + MemoryTotal: memoryMetric(item.MemTotal), + FanSpeed: metric(item.FanSpeed, "%"), + } + device := Device{ + ID: stableID(item.Type, item.BusID, strconv.FormatUint(uint64(item.Index), 10)), + Kind: KindGPU, + Vendor: item.Type, + Index: int(item.Index), + Name: item.ProductName, + Label: fmt.Sprintf("%d - %s", item.Index, item.ProductName), + BusID: item.BusID, + Metrics: metrics, + GPU: item, + } + device.Capabilities = capabilities(metrics) + for _, process := range item.Processes { + device.Processes = append(device.Processes, Process{ + PID: process.PID, + Type: process.Type, + Name: process.ProcessName, + Memory: normalizedMemoryDisplay(process.UsedMemory), + }) + } + return device +} + +func normalizeNPU(item *npu.Device) Device { + metrics := Metrics{ + Utilization: metric(item.AICore, "%"), + Temperature: metric(item.Temperature, "°C"), + Power: metric(item.PowerDraw, "W"), + MemoryUsed: memoryMetric(item.MemUsed), + MemoryTotal: memoryMetric(item.MemTotal), + } + device := Device{ + ID: fmt.Sprintf("ascend:%d:%d", item.NPUIndex, item.ChipIndex), + Kind: KindNPU, + Vendor: "ascend", + Index: int(item.Index), + NPUIndex: int(item.NPUIndex), + ChipIndex: int(item.ChipIndex), + Name: item.ProductName, + Label: fmt.Sprintf("NPU %d / Chip %d - %s", item.NPUIndex, item.ChipIndex, item.ProductName), + BusID: item.BusID, + Metrics: metrics, + NPU: item, + } + device.Capabilities = capabilities(metrics) + for _, process := range item.Processes { + device.Processes = append(device.Processes, Process{ + PID: process.PID, + Type: "NPU", + Name: process.ProcessName, + Memory: normalizedMemoryDisplay(process.UsedMemory), + }) + } + return device +} + +func normalizeXPU(item *xpu.Device) Device { + metrics := Metrics{ + Utilization: metric(item.Stats.GPUUtil, "%"), + Temperature: metric(item.Stats.Temperature, "°C"), + Power: metric(item.Stats.Power, "W"), + MemoryUsed: memoryMetric(item.Stats.MemoryUsed), + MemoryTotal: memoryMetric(item.Basic.Memory), + MemoryUtil: metric(item.Stats.MemoryUtil, "%"), + Frequency: metric(item.Stats.Frequency, "MHz"), + } + device := Device{ + ID: stableID("xpu", item.Basic.PciBdfAddress, strconv.Itoa(item.Basic.DeviceID)), + Kind: KindXPU, + Vendor: item.Basic.VendorName, + Index: item.Basic.DeviceID, + Name: item.Basic.DeviceName, + Label: fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName), + BusID: item.Basic.PciBdfAddress, + Metrics: metrics, + XPU: item, + } + device.Capabilities = capabilities(metrics) + for _, process := range item.Processes { + device.Processes = append(device.Processes, Process{ + PID: strconv.Itoa(process.PID), + Type: process.SHR, + Name: process.Command, + Memory: normalizedMemoryDisplay(process.Memory), + SharedMemory: process.SHR, + }) + } + return device +} + +func capabilities(metrics Metrics) Capabilities { + return Capabilities{ + Utilization: metrics.Utilization.Available(), + Temperature: metrics.Temperature.Available(), + Power: metrics.Power.Available(), + PowerLimit: metrics.PowerLimit.Available(), + Memory: metrics.MemoryUsed.Available() || metrics.MemoryTotal.Available(), + FanSpeed: metrics.FanSpeed.Available(), + Frequency: metrics.Frequency.Available(), + } +} + +func stableID(vendor, busID, fallback string) string { + if busID != "" && !strings.EqualFold(busID, "N/A") { + return vendor + ":" + busID + } + return vendor + ":" + fallback +} diff --git a/agent/utils/ai_tools/accelerator/provider.go b/agent/utils/ai_tools/accelerator/provider.go new file mode 100644 index 000000000..f62610399 --- /dev/null +++ b/agent/utils/ai_tools/accelerator/provider.go @@ -0,0 +1,81 @@ +package accelerator + +import ( + "context" + + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" +) + +type Provider interface { + Kind() Kind + Collect(context.Context) (*ProviderSnapshot, error) +} + +type gpuProvider struct { + client gpu.Client +} + +func (p gpuProvider) Kind() Kind { return KindGPU } + +func (p gpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { + info, err := p.client.LoadInfoContext(ctx) + if err != nil { + return nil, err + } + result := &ProviderSnapshot{ + Type: info.Type, + DriverVersion: info.DriverVersion, + CudaVersion: info.CudaVersion, + GPUs: info.Devices, + } + for index := range result.GPUs { + result.Devices = append(result.Devices, normalizeGPU(&result.GPUs[index])) + } + return result, nil +} + +type npuProvider struct { + client npu.Client +} + +func (p npuProvider) Kind() Kind { return KindNPU } + +func (p npuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { + info, err := p.client.LoadInfoContext(ctx) + if err != nil { + return nil, err + } + result := &ProviderSnapshot{ + Type: info.Type, + DriverVersion: info.DriverVersion, + NPUs: info.Devices, + } + for index := range result.NPUs { + result.Devices = append(result.Devices, normalizeNPU(&result.NPUs[index])) + } + return result, nil +} + +type xpuProvider struct { + client xpu.Client +} + +func (p xpuProvider) Kind() Kind { return KindXPU } + +func (p xpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { + info, err := p.client.LoadInfoContext(ctx) + if err != nil { + return nil, err + } + result := &ProviderSnapshot{ + Type: info.Type, + DriverVersion: info.DriverVersion, + XPUs: info.Devices, + } + for index := range result.XPUs { + result.Devices = append(result.Devices, normalizeXPU(&result.XPUs[index])) + } + return result, nil +} diff --git a/agent/utils/ai_tools/accelerator/types.go b/agent/utils/ai_tools/accelerator/types.go new file mode 100644 index 000000000..fa35a849d --- /dev/null +++ b/agent/utils/ai_tools/accelerator/types.go @@ -0,0 +1,114 @@ +package accelerator + +import ( + "errors" + + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" + "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/xpu" +) + +type Kind string + +const ( + KindGPU Kind = "gpu" + KindNPU Kind = "npu" + KindXPU Kind = "xpu" +) + +type Info struct { + Type string `json:"type"` + CudaVersion string `json:"cudaVersion"` + DriverVersion string `json:"driverVersion"` + XPUDriverVersion string `json:"xpuDriverVersion"` + GPUs []gpu.Device `json:"gpu"` + NPUs []npu.Device `json:"npu"` + XPUs []xpu.Device `json:"xpu"` +} + +type Metric struct { + Value *float64 + Unit string + Display string +} + +func (m Metric) Available() bool { + return m.Value != nil +} + +func (m Metric) ValueOrZero() float64 { + if m.Value == nil { + return 0 + } + return *m.Value +} + +type Metrics struct { + Utilization Metric + Temperature Metric + Power Metric + PowerLimit Metric + MemoryUsed Metric + MemoryTotal Metric + MemoryUtil Metric + FanSpeed Metric + Frequency Metric +} + +type Capabilities struct { + Utilization bool + Temperature bool + Power bool + PowerLimit bool + Memory bool + FanSpeed bool + Frequency bool +} + +type Process struct { + PID string + Type string + Name string + Memory string + SharedMemory string +} + +type Device struct { + ID string + Kind Kind + Vendor string + Index int + NPUIndex int + ChipIndex int + Name string + Label string + BusID string + Metrics Metrics + Capabilities Capabilities + Processes []Process + + GPU *gpu.Device `json:"-"` + NPU *npu.Device `json:"-"` + XPU *xpu.Device `json:"-"` +} + +type Snapshot struct { + Info Info + Devices []Device + DriverVersions map[string]string + Warnings []error +} + +func (s Snapshot) Warning() error { + return errors.Join(s.Warnings...) +} + +type ProviderSnapshot struct { + Type string + DriverVersion string + CudaVersion string + GPUs []gpu.Device + NPUs []npu.Device + XPUs []xpu.Device + Devices []Device +} diff --git a/agent/utils/ai_tools/gpu/amd.go b/agent/utils/ai_tools/gpu/amd.go new file mode 100644 index 000000000..36b4e537c --- /dev/null +++ b/agent/utils/ai_tools/gpu/amd.go @@ -0,0 +1,221 @@ +package gpu + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/cmd" +) + +const ( + amdSMICommand = "amd-smi" + amdSMIDefaultPath = "/opt/rocm/bin/amd-smi" +) + +type amdSMI struct { + command string +} + +func findAMDSMI() (string, bool) { + for _, command := range []string{amdSMICommand, amdSMIDefaultPath} { + if cmd.Which(command) { + return command, true + } + } + return "", false +} + +func (a amdSMI) LoadInfo(ctx context.Context) (*Info, error) { + var ( + staticData string + metricData string + processData string + staticErr error + metricErr error + processErr error + wg sync.WaitGroup + ) + + wg.Add(3) + go func() { + defer wg.Done() + staticData, staticErr = runAMDSMI(ctx, a.command, "static", "--asic", "--bus", "--driver", "--limit", "--json") + }() + go func() { + defer wg.Done() + metricData, metricErr = runAMDSMI(ctx, a.command, "metric", "--usage", "--power", "--temperature", "--mem-usage", "--fan", "--perf-level", "--json") + }() + go func() { + defer wg.Done() + processData, processErr = runAMDSMI(ctx, a.command, "process", "--general", "--json") + }() + wg.Wait() + + if staticErr != nil { + return nil, fmt.Errorf("calling %s static failed: %w", a.command, staticErr) + } + info, err := parseAMDStatic(staticData) + if err != nil { + return nil, fmt.Errorf("parsing %s static output failed: %w", a.command, err) + } + + if metricErr != nil { + global.LOG.Warnf("calling %s metric failed, metrics will be omitted: %v", a.command, metricErr) + } else if err := applyAMDMetrics(info, metricData); err != nil { + global.LOG.Warnf("parsing %s metric output failed, metrics will be omitted: %v", a.command, err) + } + if processErr != nil { + global.LOG.Warnf("calling %s process failed, process information will be omitted: %v", a.command, processErr) + } else if err := applyAMDProcesses(info, processData); err != nil { + global.LOG.Warnf("parsing %s process output failed, process information will be omitted: %v", a.command, err) + } + return info, nil +} + +func runAMDSMI(ctx context.Context, command string, args ...string) (string, error) { + cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(10*time.Second)) + return cmdMgr.RunWithStdout(command, args...) +} + +func parseAMDStatic(data string) (*Info, error) { + rows, err := decodeAMDRows(data) + if err != nil { + return nil, err + } + + info := &Info{Type: "amd"} + for _, row := range rows { + index, ok := amdGPUIndex(row) + if !ok { + continue + } + device := Device{ + Type: "amd", + Index: index, + ProductName: amdStringAt(row, "asic.market_name", "market_name", "gpu_name"), + PersistenceMode: "N/A", + BusID: amdStringAt(row, "bus.bdf", "bdf"), + DisplayActive: "N/A", + ECC: "N/A", + FanSpeed: "N/A", + Temperature: "N/A", + PerformanceState: "N/A", + PowerDraw: "N/A", + MaxPowerLimit: amdMetricAt(row, "W", + "limit.ppt0.max_power_limit", + "limit.max_power_limit", + "limit.max_power", + ), + MemUsed: "N/A", + MemTotal: "N/A", + GPUUtil: "N/A", + ComputeMode: "N/A", + MigMode: "N/A", + } + if device.ProductName == "" { + device.ProductName = "AMD GPU" + } + if device.MaxPowerLimit == "" { + device.MaxPowerLimit = "N/A" + } + if info.DriverVersion == "" { + info.DriverVersion = amdStringAt(row, "driver.version", "driver_version", "amdgpu_version") + } + info.Devices = append(info.Devices, device) + } + sort.Slice(info.Devices, func(i, j int) bool { + return info.Devices[i].Index < info.Devices[j].Index + }) + return info, nil +} + +func applyAMDMetrics(info *Info, data string) error { + rows, err := decodeAMDRows(data) + if err != nil { + return err + } + devices := amdDevicesByIndex(info) + for _, row := range rows { + index, ok := amdGPUIndex(row) + if !ok { + continue + } + device, ok := devices[index] + if !ok { + continue + } + setAMDMetric(&device.GPUUtil, row, "%", "usage.gfx_activity", "usage.gfx", "gfx_activity", "gfx_usage") + setAMDMetric(&device.Temperature, row, "°C", "temperature.hotspot", "temperature.edge", "hotspot_temperature", "gpu_temperature", "gpu_temp") + setAMDMetric(&device.PowerDraw, row, "W", "power.socket_power", "socket_power", "power_usage") + setAMDMetric(&device.MemUsed, row, "MB", "mem_usage.used_vram", "vram.used", "used_vram", "vram_used") + setAMDMetric(&device.MemTotal, row, "MB", "mem_usage.total_vram", "vram.total", "total_vram", "vram_total") + setAMDMetric(&device.FanSpeed, row, "%", "fan.speed", "fan_speed") + if value := amdStringAt(row, "perf_level", "performance_level"); value != "" { + device.PerformanceState = value + } + } + return nil +} + +func applyAMDProcesses(info *Info, data string) error { + rows, err := decodeAMDRows(data) + if err != nil { + return err + } + devices := amdDevicesByIndex(info) + for _, row := range rows { + index, ok := amdGPUIndex(row) + if !ok { + continue + } + device, ok := devices[index] + if !ok { + continue + } + processList, _ := amdValueAt(row, "process_list") + items := amdObjectList(processList) + if len(items) == 0 { + if _, ok := amdValueAt(row, "process_info"); ok { + items = amdSMIRows{row} + } + } + for _, processRow := range items { + pid := amdStringAt(processRow, "process_info.pid", "pid") + if pid == "" || strings.EqualFold(pid, "N/A") { + continue + } + device.Processes = append(device.Processes, Process{ + PID: pid, + Type: "C", + ProcessName: amdStringAt(processRow, "process_info.name", "name"), + UsedMemory: amdMetricAt(processRow, "B", + "process_info.mem_usage", + "process_info.mem", + "process_info.memory_usage.vram_mem", + "process_info.vram_mem", + "mem_usage", + ), + }) + } + } + return nil +} + +func amdDevicesByIndex(info *Info) map[uint]*Device { + devices := make(map[uint]*Device, len(info.Devices)) + for index := range info.Devices { + devices[info.Devices[index].Index] = &info.Devices[index] + } + return devices +} + +func setAMDMetric(target *string, row amdSMIRow, defaultUnit string, paths ...string) { + if value := amdMetricAt(row, defaultUnit, paths...); value != "" { + *target = value + } +} diff --git a/agent/utils/ai_tools/gpu/amd_types.go b/agent/utils/ai_tools/gpu/amd_types.go new file mode 100644 index 000000000..8b6896beb --- /dev/null +++ b/agent/utils/ai_tools/gpu/amd_types.go @@ -0,0 +1,219 @@ +package gpu + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "unicode" +) + +// amdSMIRow is one GPU entry from amd-smi JSON output. amd-smi has changed +// field names and nesting between releases, so the raw vendor response stays +// flexible and is normalized into Device in amd.go. +type amdSMIRow map[string]any + +type amdSMIRows []amdSMIRow + +// amdSMIMetricValue represents both the newer {"value": ..., "unit": ...} +// form and the scalar values returned by older amd-smi releases. +type amdSMIMetricValue struct { + Value any + Unit string +} + +func decodeAMDRows(data string) (amdSMIRows, error) { + root, err := decodeAMDJSON(data) + if err != nil { + return nil, err + } + return amdRows(root), nil +} + +func decodeAMDJSON(data string) (any, error) { + for index, char := range data { + if char != '{' && char != '[' { + continue + } + decoder := json.NewDecoder(strings.NewReader(data[index:])) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err == nil { + return value, nil + } + } + return nil, fmt.Errorf("invalid JSON output") +} + +func amdRows(value any) amdSMIRows { + switch typed := value.(type) { + case []any: + rows := make(amdSMIRows, 0, len(typed)) + for _, item := range typed { + if row, ok := item.(map[string]any); ok { + rows = append(rows, amdSMIRow(row)) + } + } + return rows + case map[string]any: + row := amdSMIRow(typed) + if _, ok := amdValueAt(row, "gpu"); ok { + return amdSMIRows{row} + } + for _, key := range []string{"gpu_data", "data"} { + if nested, ok := amdMapValue(typed, key); ok { + if rows := amdRows(nested); len(rows) > 0 { + return rows + } + } + } + } + return nil +} + +func amdObjectList(value any) amdSMIRows { + switch typed := value.(type) { + case []any: + items := make(amdSMIRows, 0, len(typed)) + for _, item := range typed { + if object, ok := item.(map[string]any); ok { + items = append(items, amdSMIRow(object)) + } + } + return items + case map[string]any: + return amdSMIRows{amdSMIRow(typed)} + default: + return nil + } +} + +func amdGPUIndex(row amdSMIRow) (uint, bool) { + value, ok := amdValueAt(row, "gpu") + if !ok { + return 0, false + } + number, err := strconv.ParseFloat(amdScalar(value), 64) + if err != nil || number < 0 { + return 0, false + } + return uint(number), true +} + +func amdStringAt(row amdSMIRow, paths ...string) string { + for _, path := range paths { + if value, ok := amdValueAt(row, path); ok { + if result := amdScalar(value); result != "" { + return result + } + } + } + return "" +} + +func amdMetricAt(row amdSMIRow, defaultUnit string, paths ...string) string { + for _, path := range paths { + value, ok := amdValueAt(row, path) + if !ok { + continue + } + if result := formatAMDMetric(value, defaultUnit); result != "" { + return result + } + } + return "" +} + +func formatAMDMetric(value any, defaultUnit string) string { + metric, ok := newAMDMetricValue(value) + if !ok { + return "" + } + formatted := amdScalar(metric.Value) + if formatted == "" || strings.EqualFold(formatted, "N/A") { + return formatted + } + if _, err := strconv.ParseFloat(formatted, 64); err != nil { + return formatted + } + if metric.Unit == "" { + metric.Unit = defaultUnit + } + if metric.Unit == "" { + return formatted + } + return formatted + " " + metric.Unit +} + +func newAMDMetricValue(value any) (amdSMIMetricValue, bool) { + metric := amdSMIMetricValue{Value: value} + object, ok := value.(map[string]any) + if !ok { + return metric, true + } + metric.Value, ok = amdMapValue(object, "value") + if !ok { + return amdSMIMetricValue{}, false + } + if unit, exists := amdMapValue(object, "unit"); exists { + metric.Unit = amdScalar(unit) + } + return metric, true +} + +func amdValueAt(row amdSMIRow, path string) (any, bool) { + var current any = map[string]any(row) + for _, key := range strings.Split(path, ".") { + object, ok := current.(map[string]any) + if !ok { + return nil, false + } + current, ok = amdMapValue(object, key) + if !ok { + return nil, false + } + } + return current, true +} + +func amdMapValue(object map[string]any, key string) (any, bool) { + wanted := normalizeAMDKey(key) + for currentKey, value := range object { + if normalizeAMDKey(currentKey) == wanted { + return value, true + } + } + return nil, false +} + +func normalizeAMDKey(value string) string { + return strings.Map(func(char rune) rune { + if unicode.IsLetter(char) || unicode.IsDigit(char) { + return unicode.ToLower(char) + } + return -1 + }, value) +} + +func amdScalar(value any) string { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case json.Number: + return typed.String() + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(typed), 'f', -1, 32) + case int: + return strconv.Itoa(typed) + case int64: + return strconv.FormatInt(typed, 10) + case uint: + return strconv.FormatUint(uint64(typed), 10) + case uint64: + return strconv.FormatUint(typed, 10) + default: + return "" + } +} diff --git a/agent/utils/ai_tools/gpu/ascend.go b/agent/utils/ai_tools/gpu/ascend.go deleted file mode 100644 index be58da4e6..000000000 --- a/agent/utils/ai_tools/gpu/ascend.go +++ /dev/null @@ -1,164 +0,0 @@ -package gpu - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" - - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common" - "github.com/1Panel-dev/1Panel/agent/utils/cmd" -) - -var ( - ascendVersionPattern = regexp.MustCompile(`(?i)\bVersion:\s*([^\s|]+)`) - ascendMemoryPattern = regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)`) -) - -type AscendSMI struct{} - -func (a AscendSMI) LoadGpuInfo() (*common.GpuInfo, error) { - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) - itemData, err := cmdMgr.RunWithStdout("npu-smi", "info") - if err != nil { - return nil, fmt.Errorf("calling npu-smi failed, %v", err) - } - return parseAscendSMI(itemData), nil -} - -func parseAscendSMI(data string) *common.GpuInfo { - info := &common.GpuInfo{Type: "ascend"} - if match := ascendVersionPattern.FindStringSubmatch(data); len(match) == 2 { - info.DriverVersion = match[1] - } - - processSection := false - deviceIndexes := make(map[uint]int) - var pending *common.GPU - - for _, line := range strings.Split(data, "\n") { - if strings.Contains(line, "Process id") && strings.Contains(line, "Process memory") { - processSection = true - pending = nil - continue - } - - cells := ascendTableCells(line) - if processSection { - if len(cells) != 4 && len(cells) != 5 { - continue - } - deviceFields := strings.Fields(cells[0]) - if len(deviceFields) == 0 { - continue - } - deviceID, err := strconv.ParseUint(deviceFields[0], 10, 64) - if err != nil { - continue - } - index, ok := deviceIndexes[uint(deviceID)] - if !ok { - continue - } - processOffset := len(cells) - 3 - info.GPUs[index].Processes = append(info.GPUs[index].Processes, common.Process{ - Pid: strings.TrimSpace(cells[processOffset]), - Type: "NPU", - ProcessName: strings.TrimSpace(cells[processOffset+1]), - UsedMemory: ascendValueWithUnit(cells[processOffset+2], "MB"), - }) - continue - } - - if len(cells) != 3 { - continue - } - if pending == nil { - fields := strings.Fields(cells[0]) - if len(fields) < 2 { - continue - } - deviceID, err := strconv.ParseUint(fields[0], 10, 64) - if err != nil { - continue - } - metrics := strings.Fields(cells[2]) - if len(metrics) < 2 { - continue - } - pending = &common.GPU{ - Type: "ascend", - Index: uint(deviceID), - ProductName: strings.Join(fields[1:], " "), - PersistenceMode: "N/A", - DisplayActive: "N/A", - ECC: "N/A", - FanSpeed: "N/A", - Temperature: ascendValueWithUnit(metrics[1], "C"), - PerformanceState: strings.TrimSpace(cells[1]), - PowerDraw: ascendValueWithUnit(metrics[0], "W"), - MaxPowerLimit: "N/A", - ComputeMode: "N/A", - MigMode: "N/A", - } - continue - } - - metrics := strings.Fields(cells[2]) - if len(metrics) == 0 { - pending = nil - continue - } - pending.BusID = strings.TrimSpace(cells[1]) - pending.GPUUtil = ascendValueWithUnit(metrics[0], "%") - pending.MemUsed, pending.MemTotal = ascendMemoryUsage(cells[2]) - deviceIndexes[pending.Index] = len(info.GPUs) - info.GPUs = append(info.GPUs, *pending) - pending = nil - } - - return info -} - -func ascendTableCells(line string) []string { - line = strings.TrimSpace(line) - if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") { - return nil - } - parts := strings.Split(line, "|") - if len(parts) < 3 { - return nil - } - cells := make([]string, 0, len(parts)-2) - for _, part := range parts[1 : len(parts)-1] { - cells = append(cells, strings.TrimSpace(part)) - } - return cells -} - -func ascendMemoryUsage(value string) (string, string) { - matches := ascendMemoryPattern.FindAllStringSubmatch(value, -1) - if len(matches) == 0 { - return "N/A", "N/A" - } - // 910B exposes both generic memory and HBM. Prefer the last memory pool - // with a non-zero capacity, which selects HBM while retaining compatibility - // with devices that only expose Memory-Usage. - selected := matches[len(matches)-1] - for i := len(matches) - 1; i >= 0; i-- { - if total, err := strconv.ParseFloat(matches[i][2], 64); err == nil && total > 0 { - selected = matches[i] - break - } - } - return selected[1] + " MB", selected[2] + " MB" -} - -func ascendValueWithUnit(value, unit string) string { - value = strings.TrimSpace(value) - if value == "" || strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") { - return "N/A" - } - return value + " " + unit -} diff --git a/agent/utils/ai_tools/gpu/client.go b/agent/utils/ai_tools/gpu/client.go new file mode 100644 index 000000000..dd1095740 --- /dev/null +++ b/agent/utils/ai_tools/gpu/client.go @@ -0,0 +1,115 @@ +package gpu + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/1Panel-dev/1Panel/agent/utils/cmd" +) + +type provider interface { + LoadInfo(context.Context) (*Info, error) +} + +type Client struct { + providers []provider +} + +type providerResult struct { + info *Info + err error +} + +func New() (bool, Client) { + client := Client{} + if cmd.Which(nvidiaSMICommand) { + client.providers = append(client.providers, nvidiaSMI{}) + } + if command, ok := findAMDSMI(); ok { + client.providers = append(client.providers, amdSMI{command: command}) + } + return len(client.providers) > 0, client +} + +func (c Client) LoadInfo() (*Info, error) { + return c.LoadInfoContext(context.Background()) +} + +func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { + results := make([]providerResult, len(c.providers)) + var wg sync.WaitGroup + for index, item := range c.providers { + wg.Add(1) + go func() { + defer wg.Done() + results[index].info, results[index].err = item.LoadInfo(ctx) + }() + } + wg.Wait() + + merged := &Info{} + var ( + errs []error + types []string + driverVersions []string + ) + for _, result := range results { + if result.err != nil { + errs = append(errs, result.err) + continue + } + if result.info == nil { + continue + } + if result.info.Type != "" { + types = append(types, result.info.Type) + } + if result.info.DriverVersion != "" { + driverVersions = append(driverVersions, formatDriverVersion(result.info.Type, result.info.DriverVersion)) + } + if result.info.CudaVersion != "" { + merged.CudaVersion = result.info.CudaVersion + } + for _, device := range result.info.Devices { + if device.Type == "" { + device.Type = result.info.Type + } + merged.Devices = append(merged.Devices, device) + } + } + + merged.Type = mergeTypes(types) + merged.DriverVersion = mergeDriverVersions(driverVersions) + if len(merged.Devices) == 0 && len(errs) > 0 { + return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...)) + } + return merged, nil +} + +func formatDriverVersion(deviceType, version string) string { + if deviceType == "" { + return version + } + return fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), version) +} + +func mergeTypes(types []string) string { + if len(types) == 1 { + return types[0] + } + if len(types) > 1 { + return "mixed" + } + return "" +} + +func mergeDriverVersions(versions []string) string { + if len(versions) == 1 { + parts := strings.SplitN(versions[0], ": ", 2) + return parts[len(parts)-1] + } + return strings.Join(versions, "；") +} diff --git a/agent/utils/ai_tools/gpu/gpu.go b/agent/utils/ai_tools/gpu/gpu.go deleted file mode 100644 index 340ffaf36..000000000 --- a/agent/utils/ai_tools/gpu/gpu.go +++ /dev/null @@ -1,151 +0,0 @@ -package gpu - -import ( - "bytes" - _ "embed" - "encoding/xml" - "errors" - "fmt" - "io" - "strings" - "sync" - "time" - - "github.com/1Panel-dev/1Panel/agent/global" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common" - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/schema" - "github.com/1Panel-dev/1Panel/agent/utils/cmd" -) - -type NvidiaSMI struct{} - -type SMI interface { - LoadGpuInfo() (*common.GpuInfo, error) -} - -func New() (bool, SMI) { - var clients []SMI - if cmd.Which("nvidia-smi") { - clients = append(clients, NvidiaSMI{}) - } - if cmd.Which("npu-smi") { - clients = append(clients, AscendSMI{}) - } - if len(clients) == 0 { - return false, nil - } - if len(clients) == 1 { - return true, clients[0] - } - return true, multiSMI{clients: clients} -} - -type multiSMI struct { - clients []SMI -} - -type smiResult struct { - info *common.GpuInfo - err error -} - -func (m multiSMI) LoadGpuInfo() (*common.GpuInfo, error) { - results := make([]smiResult, len(m.clients)) - var wg sync.WaitGroup - for index, client := range m.clients { - wg.Add(1) - go func() { - defer wg.Done() - results[index].info, results[index].err = client.LoadGpuInfo() - }() - } - wg.Wait() - - merged := &common.GpuInfo{} - var ( - errs []error - types []string - driverVersions []string - ) - for _, result := range results { - if result.err != nil { - errs = append(errs, result.err) - continue - } - if result.info == nil { - continue - } - deviceType := result.info.Type - if deviceType != "" { - types = append(types, deviceType) - } - if result.info.DriverVersion != "" { - driverVersions = append(driverVersions, fmt.Sprintf("%s: %s", strings.ToUpper(deviceType), result.info.DriverVersion)) - } - if result.info.CudaVersion != "" { - merged.CudaVersion = result.info.CudaVersion - } - for _, device := range result.info.GPUs { - if device.Type == "" { - device.Type = deviceType - } - merged.GPUs = append(merged.GPUs, device) - } - } - - if len(types) == 1 { - merged.Type = types[0] - } else if len(types) > 1 { - merged.Type = "mixed" - } - if len(driverVersions) == 1 { - parts := strings.SplitN(driverVersions[0], ": ", 2) - merged.DriverVersion = parts[len(parts)-1] - } else { - merged.DriverVersion = strings.Join(driverVersions, "；") - } - if len(merged.GPUs) == 0 && len(errs) > 0 { - return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...)) - } - return merged, nil -} - -func (n NvidiaSMI) LoadGpuInfo() (*common.GpuInfo, error) { - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) - itemData, err := cmdMgr.RunWithStdout("nvidia-smi", "-q", "-x") - if err != nil { - return nil, fmt.Errorf("calling nvidia-smi failed, %v", err) - } - data := []byte(itemData) - version := "v11" - - buf := bytes.NewBuffer(data) - decoder := xml.NewDecoder(buf) - for { - token, err := decoder.Token() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return nil, fmt.Errorf("reading token failed: %w", err) - } - d, ok := token.(xml.Directive) - if !ok { - continue - } - directive := string(d) - if !strings.HasPrefix(directive, "DOCTYPE") { - continue - } - parts := strings.Split(directive, " ") - s := strings.Trim(parts[len(parts)-1], "\" ") - if strings.HasPrefix(s, "nvsmi_device_") && strings.HasSuffix(s, ".dtd") { - version = strings.TrimSuffix(strings.TrimPrefix(s, "nvsmi_device_"), ".dtd") - } else { - global.LOG.Debugf("Cannot find schema version in %q", directive) - } - break - } - - return schema.Parse(data, version) -} diff --git a/agent/utils/ai_tools/gpu/nvidia.go b/agent/utils/ai_tools/gpu/nvidia.go new file mode 100644 index 000000000..a7193f577 --- /dev/null +++ b/agent/utils/ai_tools/gpu/nvidia.go @@ -0,0 +1,113 @@ +package gpu + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/cmd" +) + +const nvidiaSMICommand = "nvidia-smi" + +type nvidiaSMI struct{} + +func (n nvidiaSMI) LoadInfo(ctx context.Context) (*Info, error) { + cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second)) + itemData, err := cmdMgr.RunWithStdout(nvidiaSMICommand, "-q", "-x") + if err != nil { + return nil, fmt.Errorf("calling %s failed: %w", nvidiaSMICommand, err) + } + data := []byte(itemData) + version := "v11" + + buf := bytes.NewBuffer(data) + decoder := xml.NewDecoder(buf) + for { + token, err := decoder.Token() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("reading token failed: %w", err) + } + d, ok := token.(xml.Directive) + if !ok { + continue + } + directive := string(d) + if !strings.HasPrefix(directive, "DOCTYPE") { + continue + } + parts := strings.Split(directive, " ") + s := strings.Trim(parts[len(parts)-1], "\" ") + if strings.HasPrefix(s, "nvsmi_device_") && strings.HasSuffix(s, ".dtd") { + version = strings.TrimSuffix(strings.TrimPrefix(s, "nvsmi_device_"), ".dtd") + } else { + global.LOG.Debugf("Cannot find schema version in %q", directive) + } + break + } + + return parseNvidiaSMI(data, version) +} + +func parseNvidiaSMI(buf []byte, version string) (*Info, error) { + var ( + s nvidiaSMIResponse + info Info + ) + if err := xml.Unmarshal(buf, &s); err != nil { + return nil, err + } + + info.Type = "nvidia" + info.CudaVersion = s.CudaVersion + info.DriverVersion = s.DriverVersion + for i := range s.Gpu { + gpuItem := Device{ + Type: "nvidia", + Index: uint(i), + ProductName: s.Gpu[i].ProductName, + PersistenceMode: s.Gpu[i].PersistenceMode, + BusID: s.Gpu[i].ID, + DisplayActive: s.Gpu[i].DisplayActive, + ECC: s.Gpu[i].EccErrors.Volatile.DramUncorrectable, + FanSpeed: s.Gpu[i].FanSpeed, + Temperature: s.Gpu[i].Temperature.GpuTemp, + PerformanceState: s.Gpu[i].PerformanceState, + MemUsed: s.Gpu[i].FbMemoryUsage.Used, + MemTotal: s.Gpu[i].FbMemoryUsage.Total, + GPUUtil: s.Gpu[i].Utilization.GpuUtil, + ComputeMode: s.Gpu[i].ComputeMode, + MigMode: s.Gpu[i].MigMode.CurrentMig, + } + if version == "v12" || version == "v13" { + gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.PowerDraw + if gpuItem.PowerDraw == "" { + gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.InstantPowerDraw + } + gpuItem.MaxPowerLimit = s.Gpu[i].GpuPowerReadings.CurrentPowerLimit + } else { + gpuItem.PowerDraw = s.Gpu[i].PowerReadings.PowerDraw + gpuItem.MaxPowerLimit = s.Gpu[i].PowerReadings.MaxPowerLimit + } + + for _, process := range s.Gpu[i].Processes.ProcessInfo { + gpuItem.Processes = append(gpuItem.Processes, Process{ + PID: process.Pid, + Type: process.Type, + ProcessName: process.ProcessName, + UsedMemory: process.UsedMemory, + }) + } + info.Devices = append(info.Devices, gpuItem) + } + return &info, nil +} diff --git a/agent/utils/ai_tools/gpu/schema/types.go b/agent/utils/ai_tools/gpu/nvidia_types.go similarity index 99% rename from agent/utils/ai_tools/gpu/schema/types.go rename to agent/utils/ai_tools/gpu/nvidia_types.go index 7d5032427..7c0ac568e 100644 --- a/agent/utils/ai_tools/gpu/schema/types.go +++ b/agent/utils/ai_tools/gpu/nvidia_types.go @@ -1,6 +1,6 @@ -package schema +package gpu -type smi struct { +type nvidiaSMIResponse struct { AttachedGpus string `xml:"attached_gpus"` CudaVersion string `xml:"cuda_version"` DriverVersion string `xml:"driver_version"` diff --git a/agent/utils/ai_tools/gpu/schema/parser.go b/agent/utils/ai_tools/gpu/schema/parser.go deleted file mode 100644 index a80cdf541..000000000 --- a/agent/utils/ai_tools/gpu/schema/parser.go +++ /dev/null @@ -1,64 +0,0 @@ -package schema - -import ( - "encoding/xml" - - "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu/common" -) - -func Parse(buf []byte, version string) (*common.GpuInfo, error) { - var ( - s smi - info common.GpuInfo - ) - if err := xml.Unmarshal(buf, &s); err != nil { - return nil, err - } - - info.Type = "nvidia" - info.CudaVersion = s.CudaVersion - info.DriverVersion = s.DriverVersion - if len(s.Gpu) == 0 { - return &info, nil - } - for i := 0; i < len(s.Gpu); i++ { - var gpuItem common.GPU - gpuItem.Type = "nvidia" - gpuItem.Index = uint(i) - gpuItem.ProductName = s.Gpu[i].ProductName - gpuItem.PersistenceMode = s.Gpu[i].PersistenceMode - gpuItem.BusID = s.Gpu[i].ID - gpuItem.DisplayActive = s.Gpu[i].DisplayActive - gpuItem.ECC = s.Gpu[i].EccErrors.Volatile.DramUncorrectable - gpuItem.FanSpeed = s.Gpu[i].FanSpeed - - gpuItem.Temperature = s.Gpu[i].Temperature.GpuTemp - gpuItem.PerformanceState = s.Gpu[i].PerformanceState - if version == "v12" || version == "v13" { - gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.PowerDraw - if len(gpuItem.PowerDraw) == 0 { - gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.InstantPowerDraw - } - gpuItem.MaxPowerLimit = s.Gpu[i].GpuPowerReadings.CurrentPowerLimit - } else { - gpuItem.PowerDraw = s.Gpu[i].PowerReadings.PowerDraw - gpuItem.MaxPowerLimit = s.Gpu[i].PowerReadings.MaxPowerLimit - } - gpuItem.MemUsed = s.Gpu[i].FbMemoryUsage.Used - gpuItem.MemTotal = s.Gpu[i].FbMemoryUsage.Total - gpuItem.GPUUtil = s.Gpu[i].Utilization.GpuUtil - gpuItem.ComputeMode = s.Gpu[i].ComputeMode - gpuItem.MigMode = s.Gpu[i].MigMode.CurrentMig - - for _, process := range s.Gpu[i].Processes.ProcessInfo { - gpuItem.Processes = append(gpuItem.Processes, common.Process{ - Pid: process.Pid, - Type: process.Type, - ProcessName: process.ProcessName, - UsedMemory: process.UsedMemory, - }) - } - info.GPUs = append(info.GPUs, gpuItem) - } - return &info, nil -} diff --git a/agent/utils/ai_tools/gpu/common/gpu_info.go b/agent/utils/ai_tools/gpu/types.go similarity index 90% rename from agent/utils/ai_tools/gpu/common/gpu_info.go rename to agent/utils/ai_tools/gpu/types.go index af80dc5fe..495e17cb1 100644 --- a/agent/utils/ai_tools/gpu/common/gpu_info.go +++ b/agent/utils/ai_tools/gpu/types.go @@ -1,14 +1,14 @@ -package common +package gpu -type GpuInfo struct { +type Info struct { CudaVersion string `json:"cudaVersion"` DriverVersion string `json:"driverVersion"` Type string `json:"type"` - GPUs []GPU `json:"gpu"` + Devices []Device `json:"gpu"` } -type GPU struct { +type Device struct { Type string `json:"type"` Index uint `json:"index"` ProductName string `json:"productName"` @@ -31,7 +31,7 @@ type GPU struct { } type Process struct { - Pid string `json:"pid"` + PID string `json:"pid"` Type string `json:"type"` ProcessName string `json:"processName"` UsedMemory string `json:"usedMemory"` diff --git a/agent/utils/ai_tools/npu/npu.go b/agent/utils/ai_tools/npu/npu.go new file mode 100644 index 000000000..eb529e2c5 --- /dev/null +++ b/agent/utils/ai_tools/npu/npu.go @@ -0,0 +1,235 @@ +package npu + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/agent/utils/cmd" + "github.com/1Panel-dev/1Panel/agent/utils/re" +) + +const ascendSMICommand = "npu-smi" + +type Client struct{} + +func New() (bool, Client) { + return cmd.Which(ascendSMICommand), Client{} +} + +type ascendDeviceKey struct { + npuID uint + chipID uint +} + +func (c Client) LoadInfo() (*Info, error) { + return c.LoadInfoContext(context.Background()) +} + +func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { + cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second)) + itemData, err := cmdMgr.RunWithStdout(ascendSMICommand, "info") + if err != nil { + return nil, fmt.Errorf("calling %s failed: %w", ascendSMICommand, err) + } + return parseAscendSMI(itemData), nil +} + +func parseAscendSMI(data string) *Info { + info := &Info{Type: "ascend"} + if match := re.GetRegex(re.AscendVersionPattern).FindStringSubmatch(data); len(match) == 2 { + info.DriverVersion = match[1] + } + + processSection := false + deviceIndexes := make(map[ascendDeviceKey]int) + chipMetricsHeader := "" + var pending *Device + + for _, line := range strings.Split(data, "\n") { + if strings.Contains(line, "Process id") && strings.Contains(line, "Process memory") { + processSection = true + pending = nil + continue + } + + cells := ascendTableCells(line) + if len(cells) == 3 && strings.Contains(strings.ToUpper(cells[2]), "AICORE") { + chipMetricsHeader = cells[2] + continue + } + if processSection { + if len(cells) != 4 && len(cells) != 5 { + continue + } + deviceFields := strings.Fields(cells[0]) + if len(deviceFields) == 0 { + continue + } + npuID, err := strconv.ParseUint(deviceFields[0], 10, 64) + if err != nil { + continue + } + var chipID uint64 + if len(deviceFields) > 1 { + chipID, err = strconv.ParseUint(deviceFields[1], 10, 64) + if err != nil { + continue + } + } + index, ok := deviceIndexes[ascendDeviceKey{npuID: uint(npuID), chipID: uint(chipID)}] + if !ok { + continue + } + processOffset := len(cells) - 3 + info.Devices[index].Processes = append(info.Devices[index].Processes, Process{ + PID: strings.TrimSpace(cells[processOffset]), + ProcessName: strings.TrimSpace(cells[processOffset+1]), + UsedMemory: ascendValueWithUnit(cells[processOffset+2], "MB"), + }) + continue + } + + if len(cells) != 3 { + continue + } + if pending == nil { + fields := strings.Fields(cells[0]) + if len(fields) < 2 { + continue + } + deviceID, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + metrics := strings.Fields(cells[2]) + if len(metrics) < 2 { + continue + } + pending = &Device{ + Type: "ascend", + Index: uint(deviceID), + NPUIndex: uint(deviceID), + ProductName: strings.Join(fields[1:], " "), + Temperature: ascendValueWithUnit(metrics[1], "C"), + Health: strings.TrimSpace(cells[1]), + PowerDraw: ascendValueWithUnit(metrics[0], "W"), + } + if usage := ascendUsageValues(cells[2]); len(usage) > 0 { + pending.HugepagesUsed = usage[len(usage)-1][0] + pending.HugepagesTotal = usage[len(usage)-1][1] + } + continue + } + + deviceFields := strings.Fields(cells[0]) + if len(deviceFields) == 0 { + pending = nil + continue + } + chipID, err := strconv.ParseUint(deviceFields[0], 10, 64) + if err != nil { + pending = nil + continue + } + pending.ChipIndex = uint(chipID) + if len(deviceFields) > 1 { + deviceID, err := strconv.ParseUint(deviceFields[1], 10, 64) + if err != nil { + pending = nil + continue + } + pending.Index = uint(deviceID) + } + + metrics := strings.Fields(cells[2]) + if len(metrics) == 0 { + pending = nil + continue + } + pending.BusID = strings.TrimSpace(cells[1]) + pending.AICore = ascendValueWithUnit(metrics[0], "%") + pending.MemoryUsed, pending.MemoryTotal, pending.HBMUsed, pending.HBMTotal = ascendMemoryPools(cells[2], chipMetricsHeader) + pending.MemUsed, pending.MemTotal = ascendMemoryUsage(cells[2]) + deviceIndexes[ascendDeviceKey{npuID: pending.NPUIndex, chipID: pending.ChipIndex}] = len(info.Devices) + info.Devices = append(info.Devices, *pending) + pending = nil + } + + return info +} + +func ascendTableCells(line string) []string { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") { + return nil + } + parts := strings.Split(line, "|") + if len(parts) < 3 { + return nil + } + cells := make([]string, 0, len(parts)-2) + for _, part := range parts[1 : len(parts)-1] { + cells = append(cells, strings.TrimSpace(part)) + } + return cells +} + +func ascendMemoryUsage(value string) (string, string) { + usage := ascendUsageValues(value) + if len(usage) == 0 { + return "N/A", "N/A" + } + // 910B exposes both generic memory and HBM. Prefer the last memory pool + // with a non-zero capacity, which selects HBM while retaining compatibility + // with devices that only expose Memory-Usage. + selected := usage[len(usage)-1] + for i := len(usage) - 1; i >= 0; i-- { + if total, err := strconv.ParseFloat(usage[i][1], 64); err == nil && total > 0 { + selected = usage[i] + break + } + } + return selected[0] + " MB", selected[1] + " MB" +} + +func ascendMemoryPools(value, header string) (string, string, string, string) { + usage := ascendUsageValues(value) + if len(usage) == 0 { + return "", "", "", "" + } + + memoryUsed, memoryTotal := usage[0][0]+" MB", usage[0][1]+" MB" + normalizedHeader := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToUpper(header)) + if !strings.Contains(normalizedHeader, "HBMUSAGE") { + return memoryUsed, memoryTotal, "", "" + } + + hbm := usage[len(usage)-1] + return memoryUsed, memoryTotal, hbm[0] + " MB", hbm[1] + " MB" +} + +func ascendUsageValues(value string) [][2]string { + matches := re.GetRegex(re.AscendMemoryPattern).FindAllStringSubmatch(value, -1) + usage := make([][2]string, 0, len(matches)) + for _, match := range matches { + usage = append(usage, [2]string{match[1], match[2]}) + } + return usage +} + +func ascendValueWithUnit(value, unit string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if strings.EqualFold(value, "N/A") || strings.EqualFold(value, "NA") { + return "N/A" + } + if strings.HasSuffix(strings.ToUpper(value), strings.ToUpper(unit)) { + return value + } + return value + " " + unit +} diff --git a/agent/utils/ai_tools/npu/types.go b/agent/utils/ai_tools/npu/types.go new file mode 100644 index 000000000..92de01645 --- /dev/null +++ b/agent/utils/ai_tools/npu/types.go @@ -0,0 +1,36 @@ +package npu + +type Info struct { + Type string `json:"type"` + DriverVersion string `json:"driverVersion"` + Devices []Device `json:"npu"` +} + +type Device struct { + Type string `json:"type"` + Index uint `json:"index"` + NPUIndex uint `json:"npuIndex"` + ChipIndex uint `json:"chipIndex"` + ProductName string `json:"productName"` + BusID string `json:"busID"` + + Health string `json:"health"` + Temperature string `json:"temperature"` + PowerDraw string `json:"powerDraw"` + AICore string `json:"aiCore"` + MemUsed string `json:"memUsed"` + MemTotal string `json:"memTotal"` + MemoryUsed string `json:"memoryUsed"` + MemoryTotal string `json:"memoryTotal"` + HBMUsed string `json:"hbmUsed"` + HBMTotal string `json:"hbmTotal"` + HugepagesUsed string `json:"hugepagesUsed"` + HugepagesTotal string `json:"hugepagesTotal"` + Processes []Process `json:"processes"` +} + +type Process struct { + PID string `json:"pid"` + ProcessName string `json:"processName"` + UsedMemory string `json:"usedMemory"` +} diff --git a/agent/utils/ai_tools/xpu/types.go b/agent/utils/ai_tools/xpu/types.go index e78f6a7a5..94e22c9e6 100644 --- a/agent/utils/ai_tools/xpu/types.go +++ b/agent/utils/ai_tools/xpu/types.go @@ -12,7 +12,7 @@ type DeviceUtilByProcList struct { DeviceUtilByProcList []DeviceUtilByProc `json:"device_util_by_proc_list"` } -type Device struct { +type discoveryDevice struct { DeviceFunctionType string `json:"device_function_type"` DeviceID int `json:"device_id"` DeviceName string `json:"device_name"` @@ -28,8 +28,8 @@ type Device struct { DriverVersion string `json:"driver_version"` } -type DeviceInfo struct { - DeviceList []Device `json:"device_list"` +type discoveryInfo struct { + DeviceList []discoveryDevice `json:"device_list"` } type DeviceLevelMetric struct { @@ -41,3 +41,42 @@ type DeviceStats struct { DeviceID int `json:"device_id"` DeviceLevel []DeviceLevelMetric `json:"device_level"` } + +type Info struct { + Type string `json:"type"` + DriverVersion string `json:"driverVersion"` + + Devices []Device `json:"xpu"` +} + +type Device struct { + Basic Basic `json:"basic"` + Stats Stats `json:"stats"` + Processes []Process `json:"processes"` +} + +type Basic struct { + DeviceID int `json:"deviceID"` + DeviceName string `json:"deviceName"` + VendorName string `json:"vendorName"` + DriverVersion string `json:"driverVersion"` + Memory string `json:"memory"` + FreeMemory string `json:"freeMemory"` + PciBdfAddress string `json:"pciBdfAddress"` +} + +type Stats struct { + Power string `json:"power"` + GPUUtil string `json:"gpuUtil"` + Frequency string `json:"frequency"` + Temperature string `json:"temperature"` + MemoryUsed string `json:"memoryUsed"` + MemoryUtil string `json:"memoryUtil"` +} + +type Process struct { + PID int `json:"pid"` + Command string `json:"command"` + SHR string `json:"shr"` + Memory string `json:"memory"` +} diff --git a/agent/utils/ai_tools/xpu/xpu.go b/agent/utils/ai_tools/xpu/xpu.go index ea9239010..73a612c3a 100644 --- a/agent/utils/ai_tools/xpu/xpu.go +++ b/agent/utils/ai_tools/xpu/xpu.go @@ -1,6 +1,7 @@ package xpu import ( + "context" "encoding/json" "fmt" "sort" @@ -12,125 +13,29 @@ import ( "github.com/1Panel-dev/1Panel/agent/utils/cmd" ) -type XpuSMI struct{} +const xpuSMICommand = "xpu-smi" -func New() (bool, XpuSMI) { - return cmd.Which("xpu-smi"), XpuSMI{} +type Client struct{} + +func New() (bool, Client) { + return cmd.Which(xpuSMICommand), Client{} } -func (x XpuSMI) loadDeviceData(device Device, wg *sync.WaitGroup, res *[]XPUSimpleInfo, mu *sync.Mutex) { - defer wg.Done() - - var xpu XPUSimpleInfo - xpu.DeviceID = device.DeviceID - xpu.DeviceName = device.DeviceName - - var xpuData, statsData string - var xpuErr, statsErr error - - var wgCmd sync.WaitGroup - wgCmd.Add(2) - - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) - go func() { - defer wgCmd.Done() - xpuData, xpuErr = cmdMgr.RunWithStdout("xpu-smi", "discovery", "-d", strconv.Itoa(device.DeviceID), "-j") - }() - - go func() { - defer wgCmd.Done() - statsData, statsErr = cmdMgr.RunWithStdout("xpu-smi", "stats", "-d", strconv.Itoa(device.DeviceID), "-j") - }() - - wgCmd.Wait() - - if xpuErr != nil { - global.LOG.Errorf("calling xpu-smi discovery failed for device %d, %v", device.DeviceID, xpuErr) - return - } - - var info Device - if err := json.Unmarshal([]byte(xpuData), &info); err != nil { - global.LOG.Errorf("xpuData json unmarshal failed for device %d, err: %v", device.DeviceID, err) - return - } - - bytes, err := strconv.ParseInt(info.MemoryPhysicalSizeByte, 10, 64) - if err != nil { - global.LOG.Errorf("Error parsing memory size for device %d, err: %v", device.DeviceID, err) - return - } - xpu.Memory = fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024)) - - if statsErr != nil { - global.LOG.Errorf("calling xpu-smi stats failed for device %d, err: %v", device.DeviceID, statsErr) - return - } - - var stats DeviceStats - if err := json.Unmarshal([]byte(statsData), &stats); err != nil { - global.LOG.Errorf("statsData json unmarshal failed for device %d, err: %v", device.DeviceID, err) - return - } - - for _, stat := range stats.DeviceLevel { - switch stat.MetricsType { - case "XPUM_STATS_POWER": - xpu.Power = fmt.Sprintf("%.1fW", stat.Value) - case "XPUM_STATS_GPU_CORE_TEMPERATURE": - xpu.Temperature = fmt.Sprintf("%.1f°C", stat.Value) - case "XPUM_STATS_MEMORY_USED": - xpu.MemoryUsed = fmt.Sprintf("%.1fMB", stat.Value) - case "XPUM_STATS_MEMORY_UTILIZATION": - xpu.MemoryUtil = fmt.Sprintf("%.1f%%", stat.Value) - } - } - - mu.Lock() - *res = append(*res, xpu) - mu.Unlock() +func (c Client) LoadInfo() (*Info, error) { + return c.LoadInfoContext(context.Background()) } -func (x XpuSMI) LoadDashData() ([]XPUSimpleInfo, error) { - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) - data, err := cmdMgr.RunWithStdout("xpu-smi", "discovery", "-j") +func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { + cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second)) + data, err := cmdMgr.RunWithStdout(xpuSMICommand, "discovery", "-j") if err != nil { - return nil, fmt.Errorf("calling xpu-smi failed, %v", err) + return nil, fmt.Errorf("calling %s failed: %w", xpuSMICommand, err) } - - var deviceInfo DeviceInfo + var deviceInfo discoveryInfo if err := json.Unmarshal([]byte(data), &deviceInfo); err != nil { return nil, fmt.Errorf("deviceInfo json unmarshal failed, err: %w", err) } - - var res []XPUSimpleInfo - var wg sync.WaitGroup - var mu sync.Mutex - - for _, device := range deviceInfo.DeviceList { - wg.Add(1) - go x.loadDeviceData(device, &wg, &res, &mu) - } - - wg.Wait() - - sort.Slice(res, func(i, j int) bool { - return res[i].DeviceID < res[j].DeviceID - }) - return res, nil -} - -func (x XpuSMI) LoadGpuInfo() (*XpuInfo, error) { - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) - data, err := cmdMgr.RunWithStdout("xpu-smi", "discovery", "-j") - if err != nil { - return nil, fmt.Errorf("calling xpu-smi failed, %v", err) - } - var deviceInfo DeviceInfo - if err := json.Unmarshal([]byte(data), &deviceInfo); err != nil { - return nil, fmt.Errorf("deviceInfo json unmarshal failed, err: %w", err) - } - res := &XpuInfo{ + res := &Info{ Type: "xpu", } @@ -139,44 +44,49 @@ func (x XpuSMI) LoadGpuInfo() (*XpuInfo, error) { for _, device := range deviceInfo.DeviceList { wg.Add(1) - go x.loadDeviceInfo(device, &wg, res, &mu) + go c.loadDeviceInfo(ctx, device, &wg, res, &mu) } wg.Wait() - processData, err := cmdMgr.RunWithStdout("xpu-smi", "ps", "-j") + processData, err := cmdMgr.RunWithStdout(xpuSMICommand, "ps", "-j") if err != nil { - return nil, fmt.Errorf("calling xpu-smi ps failed, %s", err) - } - var psList DeviceUtilByProcList - if err := json.Unmarshal([]byte(processData), &psList); err != nil { - return nil, fmt.Errorf("processData json unmarshal failed, err: %w", err) - } - for _, ps := range psList.DeviceUtilByProcList { - process := Process{ - PID: ps.ProcessID, - Command: ps.ProcessName, - } - if ps.SharedMemSize > 0 { - process.SHR = fmt.Sprintf("%.1f MB", ps.SharedMemSize/1024) - } - if ps.MemSize > 0 { - process.Memory = fmt.Sprintf("%.1f MB", ps.MemSize/1024) - } - for index, xpu := range res.Xpu { - if xpu.Basic.DeviceID == ps.DeviceID { - res.Xpu[index].Processes = append(res.Xpu[index].Processes, process) + global.LOG.Warnf("calling xpu-smi ps failed, process information will be omitted: %v", err) + } else { + var psList DeviceUtilByProcList + if err := json.Unmarshal([]byte(processData), &psList); err != nil { + global.LOG.Warnf("processData json unmarshal failed, process information will be omitted: %v", err) + } else { + for _, ps := range psList.DeviceUtilByProcList { + process := Process{ + PID: ps.ProcessID, + Command: ps.ProcessName, + } + if ps.SharedMemSize > 0 { + process.SHR = fmt.Sprintf("%.1f MiB", ps.SharedMemSize/1024) + } + if ps.MemSize > 0 { + process.Memory = fmt.Sprintf("%.1f MiB", ps.MemSize/1024) + } + for index, xpu := range res.Devices { + if xpu.Basic.DeviceID == ps.DeviceID { + res.Devices[index].Processes = append(res.Devices[index].Processes, process) + } + } } } } + sort.Slice(res.Devices, func(i, j int) bool { + return res.Devices[i].Basic.DeviceID < res.Devices[j].Basic.DeviceID + }) return res, nil } -func (x XpuSMI) loadDeviceInfo(device Device, wg *sync.WaitGroup, res *XpuInfo, mu *sync.Mutex) { +func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg *sync.WaitGroup, res *Info, mu *sync.Mutex) { defer wg.Done() - xpu := Xpu{ + xpu := Device{ Basic: Basic{ DeviceID: device.DeviceID, DeviceName: device.DeviceName, @@ -191,15 +101,15 @@ func (x XpuSMI) loadDeviceInfo(device Device, wg *sync.WaitGroup, res *XpuInfo, var wgCmd sync.WaitGroup wgCmd.Add(2) - cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(5 * time.Second)) + cmdMgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second)) go func() { defer wgCmd.Done() - xpuData, xpuErr = cmdMgr.RunWithStdout("xpu-smi", "discovery", "-d", strconv.Itoa(device.DeviceID), "-j") + xpuData, xpuErr = cmdMgr.RunWithStdout(xpuSMICommand, "discovery", "-d", strconv.Itoa(device.DeviceID), "-j") }() go func() { defer wgCmd.Done() - statsData, statsErr = cmdMgr.RunWithStdout("xpu-smi", "stats", "-d", strconv.Itoa(device.DeviceID), "-j") + statsData, statsErr = cmdMgr.RunWithStdout(xpuSMICommand, "stats", "-d", strconv.Itoa(device.DeviceID), "-j") }() wgCmd.Wait() @@ -209,50 +119,66 @@ func (x XpuSMI) loadDeviceInfo(device Device, wg *sync.WaitGroup, res *XpuInfo, return } - var info Device + var info discoveryDevice if err := json.Unmarshal([]byte(xpuData), &info); err != nil { global.LOG.Errorf("xpuData json unmarshal failed for device %d, err: %v", device.DeviceID, err) return } - res.DriverVersion = info.DriverVersion xpu.Basic.DriverVersion = info.DriverVersion bytes, err := strconv.ParseInt(info.MemoryPhysicalSizeByte, 10, 64) if err != nil { - global.LOG.Errorf("Error parsing memory size for device %d, err: %v", device.DeviceID, err) - return + global.LOG.Warnf("Error parsing memory size for device %d, err: %v", device.DeviceID, err) + xpu.Basic.Memory = info.MemoryPhysicalSizeByte + } else { + xpu.Basic.Memory = formatMemoryBytes(bytes) + } + freeBytes, err := strconv.ParseInt(info.MemoryFreeSizeByte, 10, 64) + if err != nil { + xpu.Basic.FreeMemory = info.MemoryFreeSizeByte + } else { + xpu.Basic.FreeMemory = formatMemoryBytes(freeBytes) } - xpu.Basic.Memory = fmt.Sprintf("%.1f MB", float64(bytes)/(1024*1024)) - xpu.Basic.FreeMemory = info.MemoryFreeSizeByte if statsErr != nil { - global.LOG.Errorf("calling xpu-smi stats failed for device %d, err: %v", device.DeviceID, statsErr) - return - } - - var stats DeviceStats - if err := json.Unmarshal([]byte(statsData), &stats); err != nil { - global.LOG.Errorf("statsData json unmarshal failed for device %d, err: %v", device.DeviceID, err) - return - } - - for _, stat := range stats.DeviceLevel { - switch stat.MetricsType { - case "XPUM_STATS_POWER": - xpu.Stats.Power = fmt.Sprintf("%.1fW", stat.Value) - case "XPUM_STATS_GPU_FREQUENCY": - xpu.Stats.Frequency = fmt.Sprintf("%.1fMHz", stat.Value) - case "XPUM_STATS_GPU_CORE_TEMPERATURE": - xpu.Stats.Temperature = fmt.Sprintf("%.1f°C", stat.Value) - case "XPUM_STATS_MEMORY_USED": - xpu.Stats.MemoryUsed = fmt.Sprintf("%.1fMB", stat.Value) - case "XPUM_STATS_MEMORY_UTILIZATION": - xpu.Stats.MemoryUtil = fmt.Sprintf("%.1f%%", stat.Value) + global.LOG.Warnf("calling xpu-smi stats failed for device %d, metrics will be omitted: %v", device.DeviceID, statsErr) + } else { + var stats DeviceStats + if err := json.Unmarshal([]byte(statsData), &stats); err != nil { + global.LOG.Warnf("statsData json unmarshal failed for device %d, metrics will be omitted: %v", device.DeviceID, err) + } else { + loadStats(&xpu.Stats, stats.DeviceLevel) } } mu.Lock() - res.Xpu = append(res.Xpu, xpu) + if res.DriverVersion == "" { + res.DriverVersion = info.DriverVersion + } + res.Devices = append(res.Devices, xpu) mu.Unlock() } + +func loadStats(stats *Stats, metrics []DeviceLevelMetric) { + for _, stat := range metrics { + switch stat.MetricsType { + case "XPUM_STATS_POWER": + stats.Power = fmt.Sprintf("%.1fW", stat.Value) + case "XPUM_STATS_GPU_UTILIZATION": + stats.GPUUtil = fmt.Sprintf("%.1f%%", stat.Value) + case "XPUM_STATS_GPU_FREQUENCY": + stats.Frequency = fmt.Sprintf("%.1fMHz", stat.Value) + case "XPUM_STATS_GPU_CORE_TEMPERATURE": + stats.Temperature = fmt.Sprintf("%.1f°C", stat.Value) + case "XPUM_STATS_MEMORY_USED": + stats.MemoryUsed = fmt.Sprintf("%.1f MiB", stat.Value) + case "XPUM_STATS_MEMORY_UTILIZATION", "XPUM_STATS_MEMORY_BANDWIDTH", "XPUM_STATS_MEMORY_BANDWIDTH_UTILIZATION": + stats.MemoryUtil = fmt.Sprintf("%.1f%%", stat.Value) + } + } +} + +func formatMemoryBytes(bytes int64) string { + return fmt.Sprintf("%.1f MiB", float64(bytes)/(1024*1024)) +} diff --git a/agent/utils/ai_tools/xpu/xpu_info.go b/agent/utils/ai_tools/xpu/xpu_info.go deleted file mode 100644 index 9c7d45656..000000000 --- a/agent/utils/ai_tools/xpu/xpu_info.go +++ /dev/null @@ -1,49 +0,0 @@ -package xpu - -type XpuInfo struct { - Type string `json:"type"` - DriverVersion string `json:"driverVersion"` - - Xpu []Xpu `json:"xpu"` -} - -type Xpu struct { - Basic Basic `json:"basic"` - Stats Stats `json:"stats"` - Processes []Process `json:"processes"` -} - -type Basic struct { - DeviceID int `json:"deviceID"` - DeviceName string `json:"deviceName"` - VendorName string `json:"vendorName"` - DriverVersion string `json:"driverVersion"` - Memory string `json:"memory"` - FreeMemory string `json:"freeMemory"` - PciBdfAddress string `json:"pciBdfAddress"` -} - -type Stats struct { - Power string `json:"power"` - Frequency string `json:"frequency"` - Temperature string `json:"temperature"` - MemoryUsed string `json:"memoryUsed"` - MemoryUtil string `json:"memoryUtil"` -} - -type Process struct { - PID int `json:"pid"` - Command string `json:"command"` - SHR string `json:"shr"` - Memory string `json:"memory"` -} - -type XPUSimpleInfo struct { - DeviceID int `json:"deviceID"` - DeviceName string `json:"deviceName"` - Memory string `json:"memory"` - Temperature string `json:"temperature"` - MemoryUsed string `json:"memoryUsed"` - Power string `json:"power"` - MemoryUtil string `json:"memoryUtil"` -} diff --git a/agent/utils/re/re.go b/agent/utils/re/re.go index c0ef078da..8caa5e2b6 100644 --- a/agent/utils/re/re.go +++ b/agent/utils/re/re.go @@ -52,6 +52,9 @@ const ( NginxModulePackagePattern = `^[a-zA-Z0-9][a-zA-Z0-9+.-]*$` NginxModuleArtifactPattern = `^[a-zA-Z0-9_./+-]+\.so$` NginxModuleChecksumPattern = `^[a-fA-F0-9]{64}$` + AcceleratorMetricValuePattern = `^\s*([+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+))\s*(.*?)\s*$` + AscendVersionPattern = `(?i)\bVersion:\s*([^\s|]+)` + AscendMemoryPattern = `([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)` ) var regexMap = make(map[string]*regexp.Regexp) @@ -104,6 +107,9 @@ func Init() { NginxModulePackagePattern, NginxModuleArtifactPattern, NginxModuleChecksumPattern, + AcceleratorMetricValuePattern, + AscendVersionPattern, + AscendMemoryPattern, } for _, pattern := range patterns { diff --git a/frontend/src/api/interface/ai.ts b/frontend/src/api/interface/ai.ts index 75ba639ed..54809915c 100644 --- a/frontend/src/api/interface/ai.ts +++ b/frontend/src/api/interface/ai.ts @@ -26,9 +26,12 @@ export namespace AI { driverVersion: string; type: string; gpu: GPU[]; + npu: NPU[]; + xpuDriverVersion: string; + xpu: XpuInfo['xpu']; } export interface GPU { - type: string; + type: 'nvidia' | 'amd'; index: number; productName: string; persistenceMode: string; @@ -48,6 +51,27 @@ export namespace AI { migMode: string; processes: Process[]; } + export interface NPU { + type: 'ascend'; + index: number; + npuIndex: number; + chipIndex: number; + productName: string; + busID: string; + health: string; + temperature: string; + powerDraw: string; + aiCore: string; + memUsed: string; + memTotal: string; + memoryUsed: string; + memoryTotal: string; + hbmUsed: string; + hbmTotal: string; + hugepagesUsed: string; + hugepagesTotal: string; + processes: NPUProcess[]; + } export interface MonitorGPUSearch { productName: string; startTime: Date; @@ -60,6 +84,7 @@ export namespace AI { } export interface ChartHide { productName: string; + type?: 'gpu' | 'xpu'; process: boolean; gpu: boolean; memory: boolean; @@ -79,6 +104,7 @@ export namespace AI { memoryUsed: Array; memoryPercent: Array; speedValue: Array; + processCount: Array; gpuProcesses: Array>; } export interface GPUProcess { @@ -93,6 +119,11 @@ export namespace AI { processName: string; usedMemory: string; } + export interface NPUProcess { + pid: string; + processName: string; + usedMemory: string; + } export interface XpuInfo { type: string; @@ -118,6 +149,7 @@ export namespace AI { interface Stats { power: string; + gpuUtil: string; frequency: string; temperature: string; memoryUsed: string; diff --git a/frontend/src/api/interface/dashboard.ts b/frontend/src/api/interface/dashboard.ts index 97b8c66bc..c2ed24c67 100644 --- a/frontend/src/api/interface/dashboard.ts +++ b/frontend/src/api/interface/dashboard.ts @@ -105,6 +105,7 @@ export namespace Dashboard { diskData: Array; gpuData: Array; + npuData: Array; xpuData: Array; topCPUItems?: Array; @@ -146,20 +147,50 @@ export namespace Dashboard { export interface GPUInfo { type: string; index: number; + npuIndex: number; + chipIndex: number; productName: string; + busID: string; gpuUtil: string; temperature: string; performanceState: string; powerUsage: string; + powerDraw: string; + maxPowerLimit: string; memoryUsage: string; + memUsed: string; + memTotal: string; fanSpeed: string; } + export interface NPUInfo { + type: 'ascend'; + index: number; + npuIndex: number; + chipIndex: number; + productName: string; + busID: string; + health: string; + temperature: string; + powerDraw: string; + aiCore: string; + memUsed: string; + memTotal: string; + memoryUsed: string; + memoryTotal: string; + hbmUsed: string; + hbmTotal: string; + hugepagesUsed: string; + hugepagesTotal: string; + } + export interface XPUInfo { deviceID: number; deviceName: string; + pciBdfAddress: string; memory: string; temperature: string; + gpuUtil: string; memoryUsed: string; power: string; memoryUtil: string; diff --git a/frontend/src/api/modules/ai.ts b/frontend/src/api/modules/ai.ts index 40459250f..c0f2e0b15 100644 --- a/frontend/src/api/modules/ai.ts +++ b/frontend/src/api/modules/ai.ts @@ -39,7 +39,7 @@ export const updateBindDomain = (req: AI.BindDomain) => { }; export const loadGPUInfo = () => { - return http.get(`/ai/gpu/load`); + return http.get(`/ai/gpu/load`); }; export const getGPUOptions = () => { return http.get(`/ai/gpu/options`); diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index de4f5c4af..e5ac09231 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1453,6 +1453,8 @@ const message = { memory: 'Memory', memoryUsed: 'Memory Used', memoryTotal: 'Total Memory', + frequency: 'Frequency', + freeMemory: 'Free Memory', percent: 'Utilization', base: 'Basic Information', driverVersion: 'Driver Version', @@ -1460,6 +1462,8 @@ const message = { processMemoryUsage: 'Memory Usage', performanceStateHelper: 'From P0 (maximum performance) to P12 (minimum performance)', busID: 'Bus Address', + runtimeInfo: 'Runtime Information', + deviceInfo: 'Device Information', persistenceMode: 'Persistence Mode', enabled: 'Enabled', disabled: 'Disabled', diff --git a/frontend/src/lang/modules/es-es.ts b/frontend/src/lang/modules/es-es.ts index 10024c4cd..0d78fb9e7 100644 --- a/frontend/src/lang/modules/es-es.ts +++ b/frontend/src/lang/modules/es-es.ts @@ -1463,6 +1463,8 @@ const message = { memory: 'Memoria', memoryUsed: 'Memoria Utilizada', memoryTotal: 'Memoria Total', + frequency: 'Frecuencia', + freeMemory: 'Memoria libre', percent: 'Utilización', base: 'Información Básica', driverVersion: 'Versión del Controlador', @@ -1470,6 +1472,8 @@ const message = { processMemoryUsage: 'Uso de Memoria', performanceStateHelper: 'Desde P0 (rendimiento máximo) hasta P12 (rendimiento mínimo)', busID: 'Dirección del Bus', + runtimeInfo: 'Información de ejecución', + deviceInfo: 'Información del dispositivo', persistenceMode: 'Modo de Persistencia', enabled: 'Habilitado', disabled: 'Deshabilitado', diff --git a/frontend/src/lang/modules/fa.ts b/frontend/src/lang/modules/fa.ts index bd3412faa..efc812999 100644 --- a/frontend/src/lang/modules/fa.ts +++ b/frontend/src/lang/modules/fa.ts @@ -1439,6 +1439,8 @@ const message = { memory: 'حافظه', memoryUsed: 'حافظه استفاده شده', memoryTotal: 'حافظه کل', + frequency: 'فرکانس', + freeMemory: 'حافظه آزاد', percent: 'درصد استفاده', base: 'اطلاعات پایه', driverVersion: 'نسخه درایور', @@ -1446,6 +1448,8 @@ const message = { processMemoryUsage: 'استفاده از حافظه', performanceStateHelper: 'از P0 (حداکثر عملکرد) تا P12 (حداقل عملکرد)', busID: 'آدرس گذرگاه', + runtimeInfo: 'اطلاعات عملکرد', + deviceInfo: 'اطلاعات دستگاه', persistenceMode: 'حالت ماندگاری', enabled: 'فعال', disabled: 'غیرفعال', diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts index 20bd58d8a..104ac90a7 100644 --- a/frontend/src/lang/modules/ja.ts +++ b/frontend/src/lang/modules/ja.ts @@ -1444,6 +1444,8 @@ const message = { memory: 'メモリ', memoryUsed: '使用メモリ', memoryTotal: '総メモリ', + frequency: '周波数', + freeMemory: '空きメモリ', percent: '使用率', base: '基本情報', driverVersion: 'ドライバーバージョン', @@ -1451,6 +1453,8 @@ const message = { processMemoryUsage: 'メモリ使用量', performanceStateHelper: 'P0（最大パフォーマンス）から P12（最小パフォーマンス）まで', busID: 'バスアドレス', + runtimeInfo: '稼働情報', + deviceInfo: 'デバイス情報', persistenceMode: '永続モード', enabled: '有効', disabled: '無効', diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts index 8612bd2cf..c54187b40 100644 --- a/frontend/src/lang/modules/ko.ts +++ b/frontend/src/lang/modules/ko.ts @@ -1429,6 +1429,8 @@ const message = { memory: '메모리', memoryUsed: '사용된 메모리', memoryTotal: '전체 메모리', + frequency: '주파수', + freeMemory: '사용 가능한 메모리', percent: '사용률', base: '기본 정보', driverVersion: '드라이버 버전', @@ -1436,6 +1438,8 @@ const message = { processMemoryUsage: '메모리 사용량', performanceStateHelper: 'P0(최대 성능)부터 P12(최소 성능)까지', busID: '버스 주소', + runtimeInfo: '실행 정보', + deviceInfo: '장치 정보', persistenceMode: '지속성 모드', enabled: '활성화', disabled: '비활성화', diff --git a/frontend/src/lang/modules/lo.ts b/frontend/src/lang/modules/lo.ts index 2f49b84a3..abd869bbc 100644 --- a/frontend/src/lang/modules/lo.ts +++ b/frontend/src/lang/modules/lo.ts @@ -1425,6 +1425,8 @@ const message = { memory: 'ໜ່ວຍຄວາມຈຳ', memoryUsed: 'ໃຊ້ໄປແລ້ວ', memoryTotal: 'ທັງໝົດ', + frequency: 'ຄວາມຖີ່', + freeMemory: 'ໜ່ວຍຄວາມຈຳທີ່ຫວ່າງ', percent: 'ການນຳໃຊ້', base: 'ຂໍ້ມູນພື້ນຖານ', driverVersion: 'ເວີຊັນ Driver', @@ -1432,6 +1434,8 @@ const message = { processMemoryUsage: 'ການໃຊ້ໜ່ວຍຄວາມຈຳ', performanceStateHelper: 'ຈາກ P0 (ປະສິດທິພາບສູງສຸດ) ເຖິງ P12 (ຕ່ຳສຸດ)', busID: 'ທີ່ຢູ່ Bus', + runtimeInfo: 'ຂໍ້ມູນການເຮັດວຽກ', + deviceInfo: 'ຂໍ້ມູນອຸປະກອນ', persistenceMode: 'ໂໝດຄົງສະພາບ', enabled: 'ເປີດໃຊ້ງານ', disabled: 'ປິດໃຊ້ງານ', diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts index e8ce9f5ff..e5d58f26e 100644 --- a/frontend/src/lang/modules/ms.ts +++ b/frontend/src/lang/modules/ms.ts @@ -1466,6 +1466,8 @@ const message = { memory: 'Memori', memoryUsed: 'Memori Digunakan', memoryTotal: 'Jumlah Memori', + frequency: 'Kekerapan', + freeMemory: 'Memori Bebas', percent: 'Penggunaan', base: 'Maklumat Asas', driverVersion: 'Versi Pemacu', @@ -1473,6 +1475,8 @@ const message = { processMemoryUsage: 'Penggunaan Memori', performanceStateHelper: 'Dari P0 (prestasi maksimum) hingga P12 (prestasi minimum)', busID: 'Alamat Bas', + runtimeInfo: 'Maklumat Operasi', + deviceInfo: 'Maklumat Peranti', persistenceMode: 'Mod Kegigihan', enabled: 'Diaktifkan', disabled: 'Dilumpuhkan', diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts index 991455683..80acf5a3c 100644 --- a/frontend/src/lang/modules/pt-br.ts +++ b/frontend/src/lang/modules/pt-br.ts @@ -1464,6 +1464,8 @@ const message = { memory: 'Memória', memoryUsed: 'Memória Usada', memoryTotal: 'Memória Total', + frequency: 'Frequência', + freeMemory: 'Memória livre', percent: 'Utilização', base: 'Informações Básicas', driverVersion: 'Versão do Driver', @@ -1471,6 +1473,8 @@ const message = { processMemoryUsage: 'Uso de Memória', performanceStateHelper: 'De P0 (desempenho máximo) a P12 (desempenho mínimo)', busID: 'Endereço do Barramento', + runtimeInfo: 'Informações de execução', + deviceInfo: 'Informações do dispositivo', persistenceMode: 'Modo de Persistência', enabled: 'Habilitado', disabled: 'Desabilitado', diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts index 9f77edc01..3ac4e586e 100644 --- a/frontend/src/lang/modules/ru.ts +++ b/frontend/src/lang/modules/ru.ts @@ -1456,6 +1456,8 @@ const message = { memory: 'Память', memoryUsed: 'Использованная Память', memoryTotal: 'Общая Память', + frequency: 'Частота', + freeMemory: 'Свободная память', percent: 'Использование', base: 'Основная Информация', driverVersion: 'Версия Драйвера', @@ -1463,6 +1465,8 @@ const message = { processMemoryUsage: 'Использование Памяти', performanceStateHelper: 'От P0 (максимальная производительность) до P12 (минимальная производительность)', busID: 'Адрес Шины', + runtimeInfo: 'Рабочая информация', + deviceInfo: 'Информация об устройстве', persistenceMode: 'Режим Постоянства', enabled: 'Включено', disabled: 'Выключено', diff --git a/frontend/src/lang/modules/tr.ts b/frontend/src/lang/modules/tr.ts index 2eca75834..25ed0ee2b 100644 --- a/frontend/src/lang/modules/tr.ts +++ b/frontend/src/lang/modules/tr.ts @@ -1461,6 +1461,8 @@ const message = { memory: 'Bellek', memoryUsed: 'Kullanılan Bellek', memoryTotal: 'Toplam Bellek', + frequency: 'Frekans', + freeMemory: 'Boş Bellek', percent: 'Kullanım', base: 'Temel Bilgiler', driverVersion: 'Sürücü Sürümü', @@ -1468,6 +1470,8 @@ const message = { processMemoryUsage: 'Bellek Kullanımı', performanceStateHelper: 'P0 (maksimum performans) ile P12 (minimum performans) arası', busID: 'Veriyolu Adresi', + runtimeInfo: 'Çalışma Bilgileri', + deviceInfo: 'Cihaz Bilgileri', persistenceMode: 'Kalıcılık Modu', enabled: 'Etkin', disabled: 'Devre Dışı', diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts index 2af4dc303..ca34ab535 100644 --- a/frontend/src/lang/modules/zh-Hant.ts +++ b/frontend/src/lang/modules/zh-Hant.ts @@ -1382,6 +1382,8 @@ const message = { memory: '顯存', memoryUsed: '顯存使用', memoryTotal: '顯存總計', + frequency: '頻率', + freeMemory: '可用顯存', percent: '使用率', base: '基礎資訊', driverVersion: '驅動程式版本', @@ -1389,6 +1391,8 @@ const message = { processMemoryUsage: '顯存使用', performanceStateHelper: '從 P0 (最大效能) 到 P12 (最小效能)', busID: '匯流排位址', + runtimeInfo: '運行資訊', + deviceInfo: '設備資訊', persistenceMode: '持續模式', enabled: '開啟', disabled: '關閉', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 29b1dbd51..31a1f4c2a 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1396,6 +1396,8 @@ const message = { memory: '显存', memoryUsed: '显存使用', memoryTotal: '显存总计', + frequency: '频率', + freeMemory: '可用显存', percent: '使用率', base: '基础信息', @@ -1404,6 +1406,8 @@ const message = { processMemoryUsage: '显存使用', performanceStateHelper: '从 P0 (最大性能) 到 P12 (最小性能)', busID: '总线地址', + runtimeInfo: '运行信息', + deviceInfo: '设备信息', persistenceMode: '持续模式', enabled: '开启', disabled: '关闭', diff --git a/frontend/src/views/ai/gpu/current/index.vue b/frontend/src/views/ai/gpu/current/index.vue index 01b7a2e2f..9abfd3f0d 100644 --- a/frontend/src/views/ai/gpu/current/index.vue +++ b/frontend/src/views/ai/gpu/current/index.vue @@ -1,255 +1,410 @@