diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index bdc56f05c1..183a999926 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -46,6 +46,9 @@ const ( CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" CONTAINER_DEV_HYGON_DCU = "HYGON_DCU" CONTAINER_DEV_HYGON_DCU_HAMI = "HYGON_DCU_HAMI" + CONTAINER_DEV_ILUVATAR_GPU = "ILUVATAR_GPU" + CONTAINER_DEV_THEAD_PPU = "THEAD_PPU" + CONTAINER_DEV_KUNLUNXIN_XPU = "KUNLUNXIN_XPU" ) var ( @@ -58,6 +61,9 @@ var ( CONTAINER_DEV_VASTAITECH_GPU, CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } ) diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go index 1d98cf4abd..3898eebbb2 100644 --- a/pkg/apis/compute/isolated_device_const.go +++ b/pkg/apis/compute/isolated_device_const.go @@ -50,6 +50,9 @@ const ( VASTAITECH_VENDOR_ID = "1ec6" HYGON_VENDOR_ID = "1d94" ASCEND_VENDOR_ID = "19e5" + ILUVATAR_VENDOR_ID = "1e3e" + THEAD_VENDOR_ID = "1ded" + KUNLUNXIN_VENDOR_ID = "1d22" ) var ID_VENDOR_MAP = map[string]string{ @@ -58,6 +61,9 @@ var ID_VENDOR_MAP = map[string]string{ VASTAITECH_VENDOR_ID: "VASTAITECH", ASCEND_VENDOR_ID: "ASCEND", HYGON_VENDOR_ID: "HYGON", + ILUVATAR_VENDOR_ID: "ILUVATAR", + THEAD_VENDOR_ID: "THEAD", + KUNLUNXIN_VENDOR_ID: "KUNLUNXIN", } var VENDOR_ID_MAP = map[string]string{ @@ -66,6 +72,9 @@ var VENDOR_ID_MAP = map[string]string{ "VASTAITECH": VASTAITECH_VENDOR_ID, "ASCEND": ASCEND_VENDOR_ID, "HYGON": HYGON_VENDOR_ID, + "ILUVATAR": ILUVATAR_VENDOR_ID, + "THEAD": THEAD_VENDOR_ID, + "KUNLUNXIN": KUNLUNXIN_VENDOR_ID, } const ( @@ -91,6 +100,9 @@ var GPU_TYPES = []string{ CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_VASTAITECH_GPU, CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } var NETINT_TYPES = []string{ @@ -102,6 +114,9 @@ var VALID_CONTAINER_DEVICE_TYPES = []string{ CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU, CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } var VALID_PASSTHROUGH_TYPES = []string{ @@ -122,6 +137,9 @@ var VITRUAL_DEVICE_TYPES = []string{ CONTAINER_DEV_VASTAITECH_GPU, CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } func init() { diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 9ce5c79382..daded5385b 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -2362,6 +2362,9 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami, EnableContainerHygonDCU: options.HostOptions.EnableContainerHygonDCU, EnableContainerHygonDCUHAMI: options.HostOptions.EnableContainerHygonDCUHami, + EnableContainerIluvatarGPU: options.HostOptions.EnableContainerIluvatarGPU, + EnableContainerTHeadPPU: options.HostOptions.EnableContainerTHeadPPU, + EnableContainerKunlunxinXPU: options.HostOptions.EnableContainerKunlunxinXPU, EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, SriovNics: sriovNics, OvsOffloadNics: offloadNics, @@ -2691,6 +2694,8 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { hasNetint := false hasVasmi := false hasHygon := false + hasIluvatar := false + hasTHead := false hasNvidiasmi := false hasNpusmi := false for _, dev := range devs { @@ -2698,6 +2703,12 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { if vendorId == api.HYGON_VENDOR_ID { hasHygon = true } + if vendorId == api.ILUVATAR_VENDOR_ID { + hasIluvatar = true + } + if vendorId == api.THEAD_VENDOR_ID { + hasTHead = true + } if !utils.IsInStringArray(dev.GetSharingMode(), api.VIRTUAL_SHARING_MODES) { continue } @@ -2744,6 +2755,38 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { system_service.TELEGRAF_INPUT_CONF_BIN_PATH: options.HostOptions.HygonHySmiPath, } } + if hasIluvatar { + corexHome := options.HostOptions.IluvatarCorexHome + if corexHome == "" { + corexHome = "/usr/local/corex-4.4.0" + } + ixsmiPath := options.HostOptions.IluvatarIxsmiPath + if ixsmiPath == "" { + ixsmiPath = path.Join(corexHome, "bin", "ixsmi") + } + conf[system_service.TELEGRAF_INPUT_IXSMI] = map[string]interface{}{ + system_service.TELEGRAF_INPUT_CONF_BIN_PATH: ixsmiPath, + system_service.TELEGRAF_INPUT_CONF_LIB_PATH: path.Join(corexHome, "lib64"), + } + } + if hasTHead { + sdkHome := options.HostOptions.THeadPpuSdkHome + if sdkHome == "" { + sdkHome = "/usr/local/PPU_SDK" + } + libPath := path.Join(sdkHome, "lib64") + if !fileutils2.Exists(libPath) && fileutils2.Exists(path.Join(sdkHome, "lib")) { + libPath = path.Join(sdkHome, "lib") + } + smiPath := options.HostOptions.THeadPpuSmiPath + if smiPath == "" { + smiPath = "/usr/local/bin/ppu-smi" + } + conf[system_service.TELEGRAF_INPUT_PPUSMI] = map[string]interface{}{ + system_service.TELEGRAF_INPUT_CONF_BIN_PATH: smiPath, + system_service.TELEGRAF_INPUT_CONF_LIB_PATH: libPath, + } + } if hasNvidiasmi { conf[system_service.TELEGRAF_INPUT_NVIDIASMI] = struct{}{} } diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index 85c6d70c76..b66a948c3e 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -50,6 +50,9 @@ const ( ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU ContainerDeviceTypeHygonDcu ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU ContainerDeviceTypeHygonDcuHami ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI + ContainerDeviceTypeIluvatarGpu ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU + ContainerDeviceTypeTHeadPpu ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU + ContainerDeviceTypeKunlunxinXpu ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU ) func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) { diff --git a/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go b/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go new file mode 100644 index 0000000000..b1783c6820 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/iluvatar_gpu.go @@ -0,0 +1,266 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain the copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "os" + "path" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newIluvatarGPUManager()) +} + +type iluvatarGPUManager struct{} + +func newIluvatarGPUManager() *iluvatarGPUManager { + return &iluvatarGPUManager{} +} + +func (m *iluvatarGPUManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeIluvatarGpu +} + +func (m *iluvatarGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return getIluvatarGPUs(m) +} + +func (m *iluvatarGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *iluvatarGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + if dev.IsolatedDevice == nil { + return nil, nil, errors.Errorf("isolated device is nil") + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id) + } + gpuDev, ok := iDev.(*iluvatarGPU) + if !ok { + return nil, nil, errors.Errorf("device %s is not iluvatar gpu", dev.IsolatedDevice.Id) + } + minor := gpuDev.GetDeviceMinor() + if minor < 0 { + minor = gpuDev.GetIndex() + } + ctrDevs := []*runtimeapi.Device{} + node := iluvatarDevNode(minor) + if hygonPathExists(node) { + ctrDevs = append(ctrDevs, iluvatarDeviceSpec(node)) + } else { + log.Warningf("iluvatar container device %s not found, skip", node) + } + return ctrDevs, iluvatarCommonDevices(), nil +} + +func (m *iluvatarGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + indices := collectIluvatarVisibleIndices(devs) + return buildIluvatarExtraConfigures(indices, iluvatarCorexHome(), hygonPathExists) +} + +type iluvatarGPU struct { + manager isolated_device.IContainerDeviceManager + *BaseDevice + + memSize int + gpuIndex int + deviceMinor int + uuid string +} + +func (dev *iluvatarGPU) GetMemorySize() int { + return dev.memSize +} + +func (dev *iluvatarGPU) GetIndex() int { + return dev.gpuIndex +} + +func (dev *iluvatarGPU) GetDeviceMinor() int { + return dev.deviceMinor +} + +func (dev *iluvatarGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func iluvatarCommonDevices() []*runtimeapi.Device { + devs := []*runtimeapi.Device{} + for _, p := range collectIluvatarCommonDevicePaths(hygonPathExists) { + devs = append(devs, iluvatarDeviceSpec(p)) + } + return devs +} + +func iluvatarCorexHome() string { + home := options.HostOptions.IluvatarCorexHome + if home == "" { + return defaultIluvatarCorexHome + } + return home +} + +func iluvatarIxsmiPath() string { + p := options.HostOptions.IluvatarIxsmiPath + if p != "" { + return p + } + return path.Join(iluvatarCorexHome(), "bin", "ixsmi") +} + +func iluvatarLDLibraryPath() string { + lib64 := path.Join(iluvatarCorexHome(), "lib64") + existing := os.Getenv("LD_LIBRARY_PATH") + if existing == "" { + return lib64 + } + return existing + ":" + lib64 +} + +func runIxsmi(args ...string) (string, error) { + ixsmiPath := iluvatarIxsmiPath() + cmd := procutils.NewRemoteCommandAsFarAsPossible(ixsmiPath, args...) + cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + iluvatarLDLibraryPath()}) + out, err := cmd.Output() + if err != nil { + return string(out), errors.Wrapf(err, "ixsmi %s", strings.Join(args, " ")) + } + return string(out), nil +} + +func lookupIluvatarPCIDevice(busId, modelName string) *isolated_device.PCIDevice { + cands := iluvatarPCIAddrCandidates(busId) + for _, addr := range cands { + pciOutput, err := isolated_device.GetPCIStrByAddr(addr) + if err != nil || len(pciOutput) == 0 { + log.Warningf("iluvatar GetPCIStrByAddr %s: %v", addr, err) + continue + } + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + if modelName != "" { + dev.ModelName = modelName + } + return dev + } + fallbackAddr := busId + if len(cands) > 0 { + fallbackAddr = cands[len(cands)-1] + } + return &isolated_device.PCIDevice{ + VendorId: computeapi.ILUVATAR_VENDOR_ID, + VendorName: "ILUVATAR", + Addr: fallbackAddr, + ModelName: modelName, + DeviceName: modelName, + } +} + +func collectIluvatarVisibleIndices(devs []*hostapi.ContainerDevice) []string { + indices := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*iluvatarGPUManager); !ok { + continue + } + if dev.IsolatedDevice.Path != "" { + indices = append(indices, dev.IsolatedDevice.Path) + continue + } + if dev.IsolatedDevice.Index >= 0 { + indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index)) + } + } + return indices +} + +func getIluvatarGPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) { + ixsmiPath := iluvatarIxsmiPath() + if !hygonPathExists(ixsmiPath) { + log.Infof("iluvatar gpu probe skipped: ixsmi not found at %s", ixsmiPath) + return nil, nil + } + + tableOut, err := runIxsmi() + if err != nil { + return nil, err + } + parsed := parseIxsmiTable(tableOut) + if len(parsed) == 0 { + log.Infof("iluvatar gpu probe: no devices parsed from ixsmi table") + return nil, nil + } + + listOut, err := runIxsmi("-L") + if err != nil { + log.Warningf("ixsmi -L failed: %v", err) + } else { + parsed = mergeIluvatarProbe(parsed, parseIxsmiList(listOut)) + } + + devs := make([]isolated_device.IDevice, 0, len(parsed)) + for _, gpu := range parsed { + if gpu.ComputeMode != "" && gpu.ComputeMode != iluvatarComputeModeOK { + log.Warningf("iluvatar gpu %d compute mode %s, skip", gpu.Index, gpu.ComputeMode) + continue + } + pciDev := lookupIluvatarPCIDevice(gpu.BusId, gpu.Name) + indexStr := strconv.Itoa(gpu.Index) + minor := resolveIluvatarDeviceMinor(gpu.Index, hygonPathExists) + if minor < 0 { + log.Warningf("iluvatar gpu %d: %s not found, fallback to index for container device", gpu.Index, iluvatarDevNode(gpu.Index)) + } + dev := &iluvatarGPU{ + manager: manager, + BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1), + memSize: gpu.MemorySizeMB, + gpuIndex: gpu.Index, + deviceMinor: minor, + uuid: gpu.UUID, + } + if gpu.Name != "" { + dev.SetModelName(gpu.Name) + } + log.Infof("iluvatar gpu idx=%d minor=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s", + gpu.Index, minor, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, iluvatarDevNode(gpu.Index)) + devs = append(devs, dev) + } + if len(devs) == 0 { + return nil, nil + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi.go b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi.go new file mode 100644 index 0000000000..4a357b2090 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi.go @@ -0,0 +1,244 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain the copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "path" + "regexp" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +const ( + defaultIluvatarCorexHome = "/usr/local/corex-4.4.0" + iluvatarCorexAlias = "/usr/local/corex" + iluvatarCtlDevicePath = "/dev/itrctl" + iluvatarComputeModeOK = "Default" +) + +type parsedIluvatarGPU struct { + Index int + Name string + BusId string + MemorySizeMB int + ComputeMode string + UUID string +} + +var ( + ixsmiMemRe = regexp.MustCompile(`(\d+)\s*MiB\s*/\s*(\d+)\s*MiB`) + ixsmiListRe = regexp.MustCompile(`(?i)^GPU\s+(\d+):\s+.+\(UUID:\s*([^)]+)\)`) +) + +func splitIxsmiCols(line string) []string { + line = strings.TrimSpace(line) + line = strings.Trim(line, "|") + parts := strings.Split(line, "|") + cols := make([]string, 0, len(parts)) + for _, p := range parts { + cols = append(cols, strings.TrimSpace(p)) + } + return cols +} + +func parseIxsmiTable(output string) []*parsedIluvatarGPU { + var out []*parsedIluvatarGPU + var cur *parsedIluvatarGPU + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "+") || !strings.HasPrefix(line, "|") { + continue + } + cols := splitIxsmiCols(line) + if len(cols) < 3 { + continue + } + left := cols[0] + if strings.HasPrefix(left, "GPU") || strings.HasPrefix(left, "Fan") || + strings.HasPrefix(left, "IX-ML") || strings.HasPrefix(left, "Processes") || + strings.HasPrefix(left, "No running") { + continue + } + fields := strings.Fields(left) + if len(fields) >= 2 { + if idx, err := strconv.Atoi(fields[0]); err == nil { + name := strings.TrimSpace(strings.TrimPrefix(left, fields[0])) + cur = &parsedIluvatarGPU{ + Index: idx, + Name: name, + BusId: cols[1], + } + continue + } + } + if cur == nil { + continue + } + if m := ixsmiMemRe.FindStringSubmatch(cols[1]); len(m) == 3 { + cur.MemorySizeMB, _ = strconv.Atoi(m[2]) + } + modeFields := strings.Fields(cols[len(cols)-1]) + if len(modeFields) > 0 { + cur.ComputeMode = modeFields[len(modeFields)-1] + } + out = append(out, cur) + cur = nil + } + if cur != nil { + out = append(out, cur) + } + return out +} + +func parseIxsmiList(output string) map[int]string { + ret := map[int]string{} + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + m := ixsmiListRe.FindStringSubmatch(line) + if len(m) != 3 { + continue + } + idx, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + ret[idx] = strings.TrimSpace(m[2]) + } + return ret +} + +func mergeIluvatarProbe(table []*parsedIluvatarGPU, uuids map[int]string) []*parsedIluvatarGPU { + for _, gpu := range table { + if gpu == nil { + continue + } + if uuid, ok := uuids[gpu.Index]; ok { + gpu.UUID = uuid + } + } + return table +} + +func iluvatarPCIAddrCandidates(busId string) []string { + busId = strings.TrimSpace(busId) + if busId == "" { + return nil + } + cands := make([]string, 0, 3) + seen := map[string]bool{} + add := func(s string) { + if s == "" || seen[s] { + return + } + seen[s] = true + cands = append(cands, s) + } + add(busId) + parts := strings.Split(busId, ":") + if len(parts) == 3 { + domain, bus, fn := parts[0], parts[1], parts[2] + if len(domain) > 4 { + add(domain[len(domain)-4:] + ":" + bus + ":" + fn) + } + add(bus + ":" + fn) + } + return cands +} + +func iluvatarDevNode(minor int) string { + return fmt.Sprintf("/dev/iluvatar%d", minor) +} + +func resolveIluvatarDeviceMinor(index int, pathExists func(string) bool) int { + if pathExists != nil && pathExists(iluvatarDevNode(index)) { + return index + } + return -1 +} + +func iluvatarDeviceSpec(devPath string) *runtimeapi.Device { + return &runtimeapi.Device{ + ContainerPath: devPath, + HostPath: devPath, + Permissions: "rwm", + } +} + +func collectIluvatarCommonDevicePaths(pathExists func(string) bool) []string { + if pathExists != nil && pathExists(iluvatarCtlDevicePath) { + return []string{iluvatarCtlDevicePath} + } + return nil +} + +func normalizeIluvatarCorexHome(corexHome string) string { + if corexHome == "" { + return defaultIluvatarCorexHome + } + return corexHome +} + +func buildIluvatarRuntimeEnvs(indices []string, corexHome string) []*runtimeapi.KeyValue { + corexHome = normalizeIluvatarCorexHome(corexHome) + return []*runtimeapi.KeyValue{ + { + Key: "IX_VISIBLE_DEVICES", + Value: strings.Join(indices, ","), + }, + { + Key: "COREX_HOME", + Value: corexHome, + }, + { + Key: "LD_LIBRARY_PATH", + Value: path.Join(corexHome, "lib64"), + }, + } +} + +func buildIluvatarRuntimeMounts(corexHome string, pathExists func(string) bool) []*runtimeapi.Mount { + corexHome = normalizeIluvatarCorexHome(corexHome) + if pathExists == nil || !pathExists(corexHome) { + return nil + } + mounts := []*runtimeapi.Mount{ + { + ContainerPath: corexHome, + HostPath: corexHome, + Readonly: true, + }, + } + if path.Clean(corexHome) != path.Clean(iluvatarCorexAlias) { + mounts = append(mounts, &runtimeapi.Mount{ + ContainerPath: iluvatarCorexAlias, + HostPath: corexHome, + Readonly: true, + }) + } + return mounts +} + +func buildIluvatarExtraConfigures(indices []string, corexHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + if len(indices) == 0 { + return nil, nil + } + return buildIluvatarRuntimeEnvs(indices, corexHome), buildIluvatarRuntimeMounts(corexHome, pathExists) +} diff --git a/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go new file mode 100644 index 0000000000..8852d5acb4 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go @@ -0,0 +1,106 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain the copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIluvatarDevNode(t *testing.T) { + assert.Equal(t, "/dev/iluvatar0", iluvatarDevNode(0)) + assert.Equal(t, "/dev/iluvatar7", iluvatarDevNode(7)) +} + +func TestResolveIluvatarDeviceMinor(t *testing.T) { + exists := func(p string) bool { return p == "/dev/iluvatar3" } + assert.Equal(t, 3, resolveIluvatarDeviceMinor(3, exists)) + assert.Equal(t, -1, resolveIluvatarDeviceMinor(0, exists)) + assert.Equal(t, -1, resolveIluvatarDeviceMinor(0, nil)) +} + +func TestCollectIluvatarCommonDevicePaths(t *testing.T) { + assert.Nil(t, collectIluvatarCommonDevicePaths(nil)) + assert.Nil(t, collectIluvatarCommonDevicePaths(func(string) bool { return false })) + assert.Equal(t, []string{"/dev/itrctl"}, collectIluvatarCommonDevicePaths(func(p string) bool { + return p == "/dev/itrctl" + })) +} + +func TestIluvatarDeviceSpec(t *testing.T) { + dev := iluvatarDeviceSpec("/dev/iluvatar2") + require.NotNil(t, dev) + assert.Equal(t, "/dev/iluvatar2", dev.HostPath) + assert.Equal(t, "/dev/iluvatar2", dev.ContainerPath) + assert.Equal(t, "rwm", dev.Permissions) +} + +func TestBuildIluvatarExtraConfigures(t *testing.T) { + exists := func(p string) bool { return p == "/usr/local/corex-4.4.0" } + envs, mounts := buildIluvatarExtraConfigures([]string{"0", "1", "2", "3"}, "/usr/local/corex-4.4.0", exists) + require.Len(t, envs, 3) + assert.Equal(t, "IX_VISIBLE_DEVICES", envs[0].Key) + assert.Equal(t, "0,1,2,3", envs[0].Value) + assert.Equal(t, "COREX_HOME", envs[1].Key) + assert.Equal(t, "/usr/local/corex-4.4.0", envs[1].Value) + assert.Equal(t, "LD_LIBRARY_PATH", envs[2].Key) + assert.Equal(t, "/usr/local/corex-4.4.0/lib64", envs[2].Value) + + require.Len(t, mounts, 2) + assert.Equal(t, "/usr/local/corex-4.4.0", mounts[0].HostPath) + assert.Equal(t, "/usr/local/corex-4.4.0", mounts[0].ContainerPath) + assert.True(t, mounts[0].Readonly) + assert.Equal(t, "/usr/local/corex-4.4.0", mounts[1].HostPath) + assert.Equal(t, "/usr/local/corex", mounts[1].ContainerPath) + assert.True(t, mounts[1].Readonly) + + envs, mounts = buildIluvatarExtraConfigures(nil, "/usr/local/corex-4.4.0", exists) + assert.Nil(t, envs) + assert.Nil(t, mounts) +} + +func TestBuildIluvatarRuntimeMountsAliasOnly(t *testing.T) { + exists := func(p string) bool { return p == "/usr/local/corex" } + mounts := buildIluvatarRuntimeMounts("/usr/local/corex", exists) + require.Len(t, mounts, 1) + assert.Equal(t, "/usr/local/corex", mounts[0].HostPath) + assert.Equal(t, "/usr/local/corex", mounts[0].ContainerPath) +} + +func TestBuildIluvatarRuntimeMountsMissingHome(t *testing.T) { + assert.Nil(t, buildIluvatarRuntimeMounts("/usr/local/corex-4.4.0", func(string) bool { return false })) +} + +func TestParseIxsmiTableTwoGpus(t *testing.T) { + input := `| 0 Iluvatar BI-V150S | 00000000:26:00.0 | 500MHz 1600MHz | +| N/A 35C P0 N/A / N/A | 68MiB / 32768MiB | 0% Default | +| 1 Iluvatar BI-V150S | 00000000:29:00.0 | 500MHz 1600MHz | +| N/A 33C P0 53W / 450W | 68MiB / 32768MiB | 0% Default |` + gpus := parseIxsmiTable(input) + require.Len(t, gpus, 2) + assert.Equal(t, 0, gpus[0].Index) + assert.Equal(t, "Iluvatar BI-V150S", gpus[0].Name) + assert.Equal(t, "00000000:26:00.0", gpus[0].BusId) + assert.Equal(t, 32768, gpus[0].MemorySizeMB) + assert.Equal(t, "Default", gpus[0].ComputeMode) + assert.Equal(t, 1, gpus[1].Index) +} + +func TestIluvatarPCIAddrCandidates(t *testing.T) { + cands := iluvatarPCIAddrCandidates("00000000:26:00.0") + assert.Equal(t, []string{"00000000:26:00.0", "0000:26:00.0", "26:00.0"}, cands) +} diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go new file mode 100644 index 0000000000..761e393db6 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go @@ -0,0 +1,245 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "os" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newKunlunxinXPUManager()) +} + +type kunlunxinXPUManager struct{} + +func newKunlunxinXPUManager() *kunlunxinXPUManager { + return &kunlunxinXPUManager{} +} + +func (m *kunlunxinXPUManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeKunlunxinXpu +} + +func (m *kunlunxinXPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return getKunlunxinXPUs(m) +} + +func (m *kunlunxinXPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + if dev.IsolatedDevice == nil { + return nil, nil, errors.Errorf("isolated device is nil") + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id) + } + gpuDev, ok := iDev.(*kunlunxinXPU) + if !ok { + return nil, nil, errors.Errorf("device %s is not kunlunxin xpu", dev.IsolatedDevice.Id) + } + ctrDevs := []*runtimeapi.Device{} + node := kunlunxinXpuDevNode(gpuDev.GetIndex()) + if hygonPathExists(node) { + ctrDevs = append(ctrDevs, kunlunxinXpuDeviceSpec(node)) + } else { + log.Warningf("kunlunxin xpu container device %s not found, skip", node) + } + return ctrDevs, kunlunxinXpuCommonDevices(), nil +} + +func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + indices := collectKunlunxinXpuVisibleIndices(devs) + return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists) +} + +type kunlunxinXPU struct { + manager isolated_device.IContainerDeviceManager + *BaseDevice + + memSize int + gpuIndex int + uuid string +} + +func (dev *kunlunxinXPU) GetMemorySize() int { + return dev.memSize +} + +func (dev *kunlunxinXPU) GetIndex() int { + return dev.gpuIndex +} + +func (dev *kunlunxinXPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func kunlunxinXpuCommonDevices() []*runtimeapi.Device { + devs := []*runtimeapi.Device{} + for _, p := range collectKunlunxinXpuCommonDevicePaths(hygonPathExists) { + devs = append(devs, kunlunxinXpuDeviceSpec(p)) + } + return devs +} + +func kunlunxinXreHome() string { + home := options.HostOptions.KunlunxinXreHome + if home == "" { + return defaultKunlunxinXreHome + } + return home +} + +func kunlunxinXpuSmiPath() string { + p := options.HostOptions.KunlunxinXpuSmiPath + if p != "" { + return p + } + return defaultKunlunxinXpuSmiPath +} + +func kunlunxinXpuLDLibraryPath() string { + libDir := kunlunxinXpuLibDir(kunlunxinXreHome(), hygonPathExists) + existing := os.Getenv("LD_LIBRARY_PATH") + if existing == "" { + return libDir + } + return existing + ":" + libDir +} + +func runXpuSmi(args ...string) (string, error) { + smiPath := kunlunxinXpuSmiPath() + cmd := procutils.NewRemoteCommandAsFarAsPossible(smiPath, args...) + cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + kunlunxinXpuLDLibraryPath()}) + out, err := cmd.Output() + if err != nil { + return string(out), errors.Wrapf(err, "xpu-smi %s", strings.Join(args, " ")) + } + return string(out), nil +} + +func lookupKunlunxinPCIDevice(busId, modelName string) *isolated_device.PCIDevice { + cands := kunlunxinXpuPCIAddrCandidates(busId) + for _, addr := range cands { + pciOutput, err := isolated_device.GetPCIStrByAddr(addr) + if err != nil || len(pciOutput) == 0 { + log.Warningf("kunlunxin xpu GetPCIStrByAddr %s: %v", addr, err) + continue + } + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + if modelName != "" { + dev.ModelName = modelName + } + return dev + } + fallbackAddr := busId + if len(cands) > 0 { + fallbackAddr = cands[len(cands)-1] + } + return &isolated_device.PCIDevice{ + VendorId: computeapi.KUNLUNXIN_VENDOR_ID, + VendorName: "KUNLUNXIN", + Addr: fallbackAddr, + ModelName: modelName, + DeviceName: modelName, + } +} + +func collectKunlunxinXpuVisibleIndices(devs []*hostapi.ContainerDevice) []string { + indices := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*kunlunxinXPUManager); !ok { + continue + } + if dev.IsolatedDevice.Path != "" { + indices = append(indices, dev.IsolatedDevice.Path) + continue + } + if dev.IsolatedDevice.Index >= 0 { + indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index)) + } + } + return indices +} + +func getKunlunxinXPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) { + smiPath := kunlunxinXpuSmiPath() + if !hygonPathExists(smiPath) { + log.Infof("kunlunxin xpu probe skipped: xpu-smi not found at %s", smiPath) + return nil, nil + } + + out, err := runXpuSmi() + if err != nil { + log.Warningf("xpu-smi failed: %v", err) + return nil, nil + } + parsed := parseXpuSmiTable(out) + if len(parsed) == 0 { + log.Infof("kunlunxin xpu probe: no devices parsed from xpu-smi") + return nil, nil + } + + devs := make([]isolated_device.IDevice, 0, len(parsed)) + for _, gpu := range parsed { + pciDev := lookupKunlunxinPCIDevice(gpu.BusId, gpu.Name) + indexStr := strconv.Itoa(gpu.Index) + node := kunlunxinXpuDevNode(gpu.Index) + if !hygonPathExists(node) { + log.Warningf("kunlunxin xpu %d: %s not found, still register by index", gpu.Index, node) + } + dev := &kunlunxinXPU{ + manager: manager, + BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1), + memSize: gpu.MemorySizeMB, + gpuIndex: gpu.Index, + uuid: gpu.UUID, + } + if gpu.Name != "" { + dev.SetModelName(gpu.Name) + } + log.Infof("kunlunxin xpu idx=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s", + gpu.Index, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, node) + devs = append(devs, dev) + } + if len(devs) == 0 { + return nil, nil + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go new file mode 100644 index 0000000000..196fdc0fd5 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go @@ -0,0 +1,208 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "path" + "regexp" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +const ( + defaultKunlunxinXreHome = "/usr/local/xpu" + defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi" + kunlunxinXpuDevicePrefix = "/dev/xpu" +) + +var kunlunxinXpuCommonDevicePaths = []string{ + "/dev/xpuctrl", +} + +type parsedKunlunxinXPU struct { + Index int + Name string + BusId string + MemorySizeMB int + UUID string +} + +var ( + // | 0 P800 OAM N/A | 00000000:03:00.0 N/A | 0 | + kunlunxinXpuBusLineRe = regexp.MustCompile(`(?i)^\|\s*(\d+)\s+(.+?)\s+N/A\s+\|\s*([0-9a-f]+:[0-9a-f]+:[0-9a-f]+\.[0-9a-f]+)`) + // | N/A 35C N/A 87W / 400W | 0MiB / 98304MiB | 0% Default | + kunlunxinXpuMemUsedTotalRe = regexp.MustCompile(`(?i)(\d+)\s*MiB\s*/\s*(\d+)\s*MiB`) + kunlunxinXpuNodeRe = regexp.MustCompile(`^xpu(\d+)$`) +) + +func kunlunxinXpuDevNode(index int) string { + return fmt.Sprintf("%s%d", kunlunxinXpuDevicePrefix, index) +} + +func kunlunxinXpuDeviceSpec(devPath string) *runtimeapi.Device { + return &runtimeapi.Device{ + ContainerPath: devPath, + HostPath: devPath, + Permissions: "rwm", + } +} + +func collectKunlunxinXpuCommonDevicePaths(pathExists func(string) bool) []string { + if pathExists == nil { + return nil + } + out := make([]string, 0, len(kunlunxinXpuCommonDevicePaths)) + for _, p := range kunlunxinXpuCommonDevicePaths { + if pathExists(p) { + out = append(out, p) + } + } + return out +} + +func normalizeKunlunxinXreHome(xreHome string) string { + if xreHome == "" { + return defaultKunlunxinXreHome + } + return xreHome +} + +func kunlunxinXpuLibDir(xreHome string, pathExists func(string) bool) string { + xreHome = normalizeKunlunxinXreHome(xreHome) + candidates := []string{ + path.Join(xreHome, "so"), + path.Join(xreHome, "lib64"), + path.Join(xreHome, "lib"), + } + if pathExists == nil { + return candidates[0] + } + for _, p := range candidates { + if pathExists(p) { + return p + } + } + return candidates[0] +} + +func parseXpuSmiTable(output string) []*parsedKunlunxinXPU { + var out []*parsedKunlunxinXPU + lines := strings.Split(output, "\n") + for i := 0; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + m := kunlunxinXpuBusLineRe.FindStringSubmatch(line) + if len(m) != 4 { + continue + } + idx, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + name := strings.TrimSpace(m[2]) + busId := strings.TrimSpace(m[3]) + mem := 0 + // memory total is on the next data row + for j := i + 1; j < len(lines) && j <= i+3; j++ { + memLine := strings.TrimSpace(lines[j]) + mm := kunlunxinXpuMemUsedTotalRe.FindStringSubmatch(memLine) + if len(mm) == 3 { + mem, _ = strconv.Atoi(mm[2]) + break + } + } + out = append(out, &parsedKunlunxinXPU{ + Index: idx, + Name: name, + BusId: busId, + MemorySizeMB: mem, + UUID: busId, + }) + } + return out +} + +func kunlunxinXpuPCIAddrCandidates(busId string) []string { + busId = strings.TrimSpace(busId) + if busId == "" { + return nil + } + cands := make([]string, 0, 3) + seen := map[string]bool{} + add := func(s string) { + if s == "" || seen[s] { + return + } + seen[s] = true + cands = append(cands, s) + } + add(busId) + parts := strings.Split(busId, ":") + if len(parts) == 3 { + domain, bus, fn := parts[0], parts[1], parts[2] + if len(domain) > 4 { + add(domain[len(domain)-4:] + ":" + bus + ":" + fn) + } + add(bus + ":" + fn) + } + return cands +} + +func parseKunlunxinXpuNodeIndex(name string) (int, bool) { + m := kunlunxinXpuNodeRe.FindStringSubmatch(name) + if len(m) != 2 { + return 0, false + } + idx, err := strconv.Atoi(m[1]) + if err != nil { + return 0, false + } + return idx, true +} + +func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists func(string) bool) []*runtimeapi.KeyValue { + xreHome = normalizeKunlunxinXreHome(xreHome) + visible := strings.Join(indices, ",") + return []*runtimeapi.KeyValue{ + {Key: "XPU_VISIBLE_DEVICES", Value: visible}, + {Key: "LD_LIBRARY_PATH", Value: kunlunxinXpuLibDir(xreHome, pathExists)}, + } +} + +func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount { + xreHome = normalizeKunlunxinXreHome(xreHome) + if pathExists == nil || !pathExists(xreHome) { + return nil + } + return []*runtimeapi.Mount{ + { + ContainerPath: xreHome, + HostPath: xreHome, + Readonly: true, + }, + } +} + +func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + if len(indices) == 0 { + return nil, nil + } + return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists) +} diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go new file mode 100644 index 0000000000..dbbe0b4473 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go @@ -0,0 +1,111 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "yunion.io/x/onecloud/pkg/hostman/options" +) + +func TestKunlunxinXpuDevNodeUsesIndex(t *testing.T) { + assert.Equal(t, "/dev/xpu0", kunlunxinXpuDevNode(0)) + assert.Equal(t, "/dev/xpu7", kunlunxinXpuDevNode(7)) +} + +func TestCollectKunlunxinXpuCommonDevicePaths(t *testing.T) { + assert.Nil(t, collectKunlunxinXpuCommonDevicePaths(nil)) + assert.Empty(t, collectKunlunxinXpuCommonDevicePaths(func(string) bool { return false })) + assert.Equal(t, []string{"/dev/xpuctrl"}, collectKunlunxinXpuCommonDevicePaths(func(string) bool { return true })) +} + +func TestParseXpuSmiTable(t *testing.T) { + input := `Thu Sep 10 08:18:37 2026 ++-----------------------------------------------------------------------------+ +| XPU-SMI Driver Version: 5.19.0.0 XPU-RT Version: 5.19.0 | +|-------------------------------+----------------------+----------------------+ +| XPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | XPU-Util Compute M. | +| | L3-Usage | SR-IOV M. | +|===============================+======================+======================| +| 0 P800 OAM N/A | 00000000:03:00.0 N/A | 0 | +| N/A 35C N/A 87W / 400W | 0MiB / 98304MiB | 0% Default | +| | 0MiB / 96MiB | Disabled | ++-------------------------------+----------------------+----------------------+ +| 1 P800 OAM N/A | 00000000:05:00.0 N/A | 0 | +| N/A 37C N/A 84W / 400W | 0MiB / 98304MiB | 0% Default | +| | 0MiB / 96MiB | Disabled | ++-------------------------------+----------------------+----------------------+ +| 7 P800 OAM N/A | 00000000:A5:00.0 N/A | 0 | +| N/A 35C N/A 85W / 400W | 0MiB / 98304MiB | 0% Default | +| | 0MiB / 96MiB | Disabled | ++-------------------------------+----------------------+----------------------+ +` + gpus := parseXpuSmiTable(input) + require.Len(t, gpus, 3) + assert.Equal(t, 0, gpus[0].Index) + assert.Equal(t, "P800 OAM", gpus[0].Name) + assert.Equal(t, "00000000:03:00.0", gpus[0].BusId) + assert.Equal(t, "00000000:03:00.0", gpus[0].UUID) + assert.Equal(t, 98304, gpus[0].MemorySizeMB) + assert.Equal(t, 1, gpus[1].Index) + assert.Equal(t, "00000000:05:00.0", gpus[1].BusId) + assert.Equal(t, 7, gpus[2].Index) + assert.Equal(t, "00000000:A5:00.0", gpus[2].BusId) +} + +func TestKunlunxinXpuPCIAddrCandidates(t *testing.T) { + cands := kunlunxinXpuPCIAddrCandidates("00000000:a8:00.0") + assert.Equal(t, []string{"00000000:a8:00.0", "0000:a8:00.0", "a8:00.0"}, cands) +} + +func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) { + exists := func(p string) bool { + return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib" + } + envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists) + require.Len(t, envs, 2) + assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key) + assert.Equal(t, "0,3", envs[0].Value) + assert.Equal(t, "LD_LIBRARY_PATH", envs[1].Key) + assert.Equal(t, "/usr/local/xpu/lib", envs[1].Value) + require.Len(t, mounts, 1) + assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath) + assert.True(t, mounts[0].Readonly) + + envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists) + assert.Nil(t, envs) + assert.Nil(t, mounts) +} + +func TestParseKunlunxinXpuNodeIndex(t *testing.T) { + idx, ok := parseKunlunxinXpuNodeIndex("xpu3") + assert.True(t, ok) + assert.Equal(t, 3, idx) + _, ok = parseKunlunxinXpuNodeIndex("xpuctrl") + assert.False(t, ok) +} + +func TestKunlunxinXpuSmiPathDefault(t *testing.T) { + orig := options.HostOptions.KunlunxinXpuSmiPath + defer func() { options.HostOptions.KunlunxinXpuSmiPath = orig }() + options.HostOptions.KunlunxinXpuSmiPath = "" + assert.Equal(t, "/usr/local/bin/xpu-smi", kunlunxinXpuSmiPath()) + options.HostOptions.KunlunxinXpuSmiPath = "/opt/bin/xpu-smi" + assert.Equal(t, "/opt/bin/xpu-smi", kunlunxinXpuSmiPath()) +} diff --git a/pkg/hostman/isolated_device/container_device/thead_ppu.go b/pkg/hostman/isolated_device/container_device/thead_ppu.go new file mode 100644 index 0000000000..affe23f0ca --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/thead_ppu.go @@ -0,0 +1,259 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "os" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newTHeadPPUManager()) +} + +type tHeadPPUManager struct{} + +func newTHeadPPUManager() *tHeadPPUManager { + return &tHeadPPUManager{} +} + +func (m *tHeadPPUManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeTHeadPpu +} + +func (m *tHeadPPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return getTHeadPPUs(m) +} + +func (m *tHeadPPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *tHeadPPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + if dev.IsolatedDevice == nil { + return nil, nil, errors.Errorf("isolated device is nil") + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id) + } + gpuDev, ok := iDev.(*tHeadPPU) + if !ok { + return nil, nil, errors.Errorf("device %s is not t-head ppu", dev.IsolatedDevice.Id) + } + ctrDevs := []*runtimeapi.Device{} + node := tHeadPpuDevNode(gpuDev.GetIndex()) + if hygonPathExists(node) { + ctrDevs = append(ctrDevs, tHeadPpuDeviceSpec(node)) + } else { + log.Warningf("t-head ppu container device %s not found, skip", node) + } + return ctrDevs, tHeadPpuCommonDevices(), nil +} + +func (m *tHeadPPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + indices := collectTHeadPpuVisibleIndices(devs) + return buildTHeadPpuExtraConfigures(indices, tHeadPpuSdkHome(), hygonPathExists) +} + +type tHeadPPU struct { + manager isolated_device.IContainerDeviceManager + *BaseDevice + + memSize int + gpuIndex int + uuid string +} + +func (dev *tHeadPPU) GetMemorySize() int { + return dev.memSize +} + +func (dev *tHeadPPU) GetIndex() int { + return dev.gpuIndex +} + +func (dev *tHeadPPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func tHeadPpuCommonDevices() []*runtimeapi.Device { + devs := []*runtimeapi.Device{} + for _, p := range collectTHeadPpuCommonDevicePaths(hygonPathExists) { + devs = append(devs, tHeadPpuDeviceSpec(p)) + } + return devs +} + +func tHeadPpuSdkHome() string { + home := options.HostOptions.THeadPpuSdkHome + if home == "" { + return defaultTHeadPpuSdkHome + } + return home +} + +func tHeadPpuSmiPath() string { + p := options.HostOptions.THeadPpuSmiPath + if p != "" { + return p + } + return defaultTHeadPpuSmiPath +} + +func tHeadPpuLDLibraryPath() string { + libDir := tHeadPpuLibDir(tHeadPpuSdkHome(), hygonPathExists) + existing := os.Getenv("LD_LIBRARY_PATH") + if existing == "" { + return libDir + } + return existing + ":" + libDir +} + +func runPpuSmi(args ...string) (string, error) { + smiPath := tHeadPpuSmiPath() + cmd := procutils.NewRemoteCommandAsFarAsPossible(smiPath, args...) + cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + tHeadPpuLDLibraryPath()}) + out, err := cmd.Output() + if err != nil { + return string(out), errors.Wrapf(err, "ppu-smi %s", strings.Join(args, " ")) + } + return string(out), nil +} + +func lookupTHeadPCIDevice(busId, modelName string) *isolated_device.PCIDevice { + cands := tHeadPpuPCIAddrCandidates(busId) + for _, addr := range cands { + pciOutput, err := isolated_device.GetPCIStrByAddr(addr) + if err != nil || len(pciOutput) == 0 { + log.Warningf("t-head ppu GetPCIStrByAddr %s: %v", addr, err) + continue + } + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + if modelName != "" { + dev.ModelName = modelName + } + return dev + } + fallbackAddr := busId + if len(cands) > 0 { + fallbackAddr = cands[len(cands)-1] + } + return &isolated_device.PCIDevice{ + VendorId: computeapi.THEAD_VENDOR_ID, + VendorName: "THEAD", + Addr: fallbackAddr, + ModelName: modelName, + DeviceName: modelName, + } +} + +func collectTHeadPpuVisibleIndices(devs []*hostapi.ContainerDevice) []string { + indices := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + continue + } + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*tHeadPPUManager); !ok { + continue + } + if dev.IsolatedDevice.Path != "" { + indices = append(indices, dev.IsolatedDevice.Path) + continue + } + if dev.IsolatedDevice.Index >= 0 { + indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index)) + } + } + return indices +} + +func getTHeadPPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) { + smiPath := tHeadPpuSmiPath() + if !hygonPathExists(smiPath) { + log.Infof("t-head ppu probe skipped: ppu-smi not found at %s", smiPath) + return nil, nil + } + + parsed := []*parsedTHeadPPU{} + queryOut, err := runPpuSmi("--query-ppu=index,name,uuid,pci.bus_id,memory.total", "--format=csv,noheader,nounits") + if err != nil { + log.Warningf("ppu-smi --query-ppu failed: %v", err) + } else { + parsed = parsePpuSmiQueryCSV(queryOut) + } + + listOut, err := runPpuSmi("-L") + if err != nil { + log.Warningf("ppu-smi -L failed: %v", err) + } else { + parsed = mergeTHeadPpuProbe(parsed, parsePpuSmiList(listOut)) + if len(parsed) == 0 { + for idx, uuid := range parsePpuSmiList(listOut) { + parsed = append(parsed, &parsedTHeadPPU{Index: idx, UUID: uuid}) + } + } + } + + if len(parsed) == 0 { + log.Infof("t-head ppu probe: no devices parsed from ppu-smi") + return nil, nil + } + + devs := make([]isolated_device.IDevice, 0, len(parsed)) + for _, gpu := range parsed { + pciDev := lookupTHeadPCIDevice(gpu.BusId, gpu.Name) + indexStr := strconv.Itoa(gpu.Index) + node := tHeadPpuDevNode(gpu.Index) + if !hygonPathExists(node) { + log.Warningf("t-head ppu %d: %s not found, still register by index", gpu.Index, node) + } + dev := &tHeadPPU{ + manager: manager, + BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1), + memSize: gpu.MemorySizeMB, + gpuIndex: gpu.Index, + uuid: gpu.UUID, + } + if gpu.Name != "" { + dev.SetModelName(gpu.Name) + } + log.Infof("t-head ppu idx=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s", + gpu.Index, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, node) + devs = append(devs, dev) + } + if len(devs) == 0 { + return nil, nil + } + return devs, nil +} diff --git a/pkg/hostman/isolated_device/container_device/thead_ppusmi.go b/pkg/hostman/isolated_device/container_device/thead_ppusmi.go new file mode 100644 index 0000000000..a912de5ff2 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/thead_ppusmi.go @@ -0,0 +1,242 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "path" + "regexp" + "strconv" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +const ( + defaultTHeadPpuSdkHome = "/usr/local/PPU_SDK" + defaultTHeadPpuSmiPath = "/usr/local/bin/ppu-smi" + tHeadPpuDevicePrefix = "/dev/alixpu_ppu" +) + +var tHeadPpuCommonDevicePaths = []string{ + "/dev/alixpu", + "/dev/alixpu_ctl", + "/dev/alixpu_sep", +} + +type parsedTHeadPPU struct { + Index int + Name string + BusId string + MemorySizeMB int + UUID string +} + +var ( + tHeadPpuListRe = regexp.MustCompile(`(?i)^PPU\s+(\d+):\s+.+\(UUID:\s*([^)]+)\)`) + tHeadPpuMemRe = regexp.MustCompile(`(\d+)`) + tHeadPpuNodeRe = regexp.MustCompile(`^alixpu_ppu(\d+)$`) + tHeadPpuQueryRe = regexp.MustCompile(`index,name,uuid,pci\.bus_id,memory\.total`) +) + +func tHeadPpuDevNode(index int) string { + return fmt.Sprintf("%s%d", tHeadPpuDevicePrefix, index) +} + +func tHeadPpuDeviceSpec(devPath string) *runtimeapi.Device { + return &runtimeapi.Device{ + ContainerPath: devPath, + HostPath: devPath, + Permissions: "rwm", + } +} + +func collectTHeadPpuCommonDevicePaths(pathExists func(string) bool) []string { + if pathExists == nil { + return nil + } + out := make([]string, 0, len(tHeadPpuCommonDevicePaths)) + for _, p := range tHeadPpuCommonDevicePaths { + if pathExists(p) { + out = append(out, p) + } + } + return out +} + +func normalizeTHeadPpuSdkHome(sdkHome string) string { + if sdkHome == "" { + return defaultTHeadPpuSdkHome + } + return sdkHome +} + +func tHeadPpuLibDir(sdkHome string, pathExists func(string) bool) string { + sdkHome = normalizeTHeadPpuSdkHome(sdkHome) + lib64 := path.Join(sdkHome, "lib64") + if pathExists == nil || pathExists(lib64) { + return lib64 + } + return path.Join(sdkHome, "lib") +} + +func parsePpuSmiQueryCSV(output string) []*parsedTHeadPPU { + var out []*parsedTHeadPPU + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if tHeadPpuQueryRe.MatchString(strings.ToLower(strings.ReplaceAll(line, " ", ""))) { + continue + } + if strings.HasPrefix(strings.ToLower(line), "timestamp") { + continue + } + cols := splitCSVLine(line) + if len(cols) < 5 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(cols[0])) + if err != nil { + continue + } + mem := 0 + if m := tHeadPpuMemRe.FindStringSubmatch(cols[4]); len(m) == 2 { + mem, _ = strconv.Atoi(m[1]) + } + out = append(out, &parsedTHeadPPU{ + Index: idx, + Name: strings.TrimSpace(cols[1]), + UUID: strings.TrimSpace(cols[2]), + BusId: strings.TrimSpace(cols[3]), + MemorySizeMB: mem, + }) + } + return out +} + +func splitCSVLine(line string) []string { + parts := strings.Split(line, ",") + cols := make([]string, 0, len(parts)) + for _, p := range parts { + cols = append(cols, strings.TrimSpace(p)) + } + return cols +} + +func parsePpuSmiList(output string) map[int]string { + ret := map[int]string{} + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + m := tHeadPpuListRe.FindStringSubmatch(line) + if len(m) != 3 { + continue + } + idx, err := strconv.Atoi(m[1]) + if err != nil { + continue + } + ret[idx] = strings.TrimSpace(m[2]) + } + return ret +} + +func mergeTHeadPpuProbe(table []*parsedTHeadPPU, uuids map[int]string) []*parsedTHeadPPU { + for _, gpu := range table { + if gpu == nil { + continue + } + if gpu.UUID != "" { + continue + } + if uuid, ok := uuids[gpu.Index]; ok { + gpu.UUID = uuid + } + } + return table +} + +func tHeadPpuPCIAddrCandidates(busId string) []string { + busId = strings.TrimSpace(busId) + if busId == "" { + return nil + } + cands := make([]string, 0, 3) + seen := map[string]bool{} + add := func(s string) { + if s == "" || seen[s] { + return + } + seen[s] = true + cands = append(cands, s) + } + add(busId) + parts := strings.Split(busId, ":") + if len(parts) == 3 { + domain, bus, fn := parts[0], parts[1], parts[2] + if len(domain) > 4 { + add(domain[len(domain)-4:] + ":" + bus + ":" + fn) + } + add(bus + ":" + fn) + } + return cands +} + +func parseTHeadPpuNodeIndex(name string) (int, bool) { + m := tHeadPpuNodeRe.FindStringSubmatch(name) + if len(m) != 2 { + return 0, false + } + idx, err := strconv.Atoi(m[1]) + if err != nil { + return 0, false + } + return idx, true +} + +func buildTHeadPpuRuntimeEnvs(indices []string, sdkHome string, pathExists func(string) bool) []*runtimeapi.KeyValue { + sdkHome = normalizeTHeadPpuSdkHome(sdkHome) + visible := strings.Join(indices, ",") + return []*runtimeapi.KeyValue{ + {Key: "CUDA_VISIBLE_DEVICES", Value: visible}, + {Key: "NVIDIA_VISIBLE_DEVICES", Value: visible}, + {Key: "LD_LIBRARY_PATH", Value: tHeadPpuLibDir(sdkHome, pathExists)}, + } +} + +func buildTHeadPpuRuntimeMounts(sdkHome string, pathExists func(string) bool) []*runtimeapi.Mount { + sdkHome = normalizeTHeadPpuSdkHome(sdkHome) + if pathExists == nil || !pathExists(sdkHome) { + return nil + } + return []*runtimeapi.Mount{ + { + ContainerPath: sdkHome, + HostPath: sdkHome, + Readonly: true, + }, + } +} + +func buildTHeadPpuExtraConfigures(indices []string, sdkHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + if len(indices) == 0 { + return nil, nil + } + return buildTHeadPpuRuntimeEnvs(indices, sdkHome, pathExists), buildTHeadPpuRuntimeMounts(sdkHome, pathExists) +} diff --git a/pkg/hostman/isolated_device/container_device/thead_ppusmi_test.go b/pkg/hostman/isolated_device/container_device/thead_ppusmi_test.go new file mode 100644 index 0000000000..e82775c3d2 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/thead_ppusmi_test.go @@ -0,0 +1,108 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "yunion.io/x/onecloud/pkg/hostman/options" +) + +func TestTHeadPpuDevNodeUsesIndexNotMinor(t *testing.T) { + assert.Equal(t, "/dev/alixpu_ppu0", tHeadPpuDevNode(0)) + assert.Equal(t, "/dev/alixpu_ppu3", tHeadPpuDevNode(3)) + assert.NotEqual(t, "/dev/alixpu_ppu4", tHeadPpuDevNode(3)) +} + +func TestCollectTHeadPpuCommonDevicePaths(t *testing.T) { + assert.Nil(t, collectTHeadPpuCommonDevicePaths(nil)) + assert.Empty(t, collectTHeadPpuCommonDevicePaths(func(string) bool { return false })) + assert.Equal(t, []string{ + "/dev/alixpu", + "/dev/alixpu_ctl", + "/dev/alixpu_sep", + }, collectTHeadPpuCommonDevicePaths(func(string) bool { return true })) + assert.Equal(t, []string{"/dev/alixpu", "/dev/alixpu_sep"}, collectTHeadPpuCommonDevicePaths(func(p string) bool { + return p != "/dev/alixpu_ctl" + })) +} + +func TestParsePpuSmiQueryCSV(t *testing.T) { + input := `index, name, uuid, pci.bus_id, memory.total [MiB] +0, PPU-ZW810E, GPU-019e2226-4211-0208-0000-000000ab261d, 00000000:a8:00.0, 98304 MiB +1, PPU-ZW810E, GPU-019e2226-84c1-0200-0000-0000c020e701, 00000000:a7:00.0, 98304` + gpus := parsePpuSmiQueryCSV(input) + require.Len(t, gpus, 2) + assert.Equal(t, 0, gpus[0].Index) + assert.Equal(t, "PPU-ZW810E", gpus[0].Name) + assert.Equal(t, "GPU-019e2226-4211-0208-0000-000000ab261d", gpus[0].UUID) + assert.Equal(t, "00000000:a8:00.0", gpus[0].BusId) + assert.Equal(t, 98304, gpus[0].MemorySizeMB) + assert.Equal(t, 1, gpus[1].Index) +} + +func TestParsePpuSmiList(t *testing.T) { + input := `PPU 0: PPU-ZW810E (UUID: GPU-019ea108-c110-0828-0000-0000c07e1a46) +PPU 1: PPU (UUID: GPU-019ea108-c120-040c-0000-0000c0267f1e)` + uuids := parsePpuSmiList(input) + assert.Equal(t, "GPU-019ea108-c110-0828-0000-0000c07e1a46", uuids[0]) + assert.Equal(t, "GPU-019ea108-c120-040c-0000-0000c0267f1e", uuids[1]) +} + +func TestTHeadPpuPCIAddrCandidates(t *testing.T) { + cands := tHeadPpuPCIAddrCandidates("00000000:a8:00.0") + assert.Equal(t, []string{"00000000:a8:00.0", "0000:a8:00.0", "a8:00.0"}, cands) +} + +func TestBuildTHeadPpuExtraConfigures(t *testing.T) { + exists := func(p string) bool { + return p == "/usr/local/PPU_SDK" || p == "/usr/local/PPU_SDK/lib" + } + envs, mounts := buildTHeadPpuExtraConfigures([]string{"0", "3"}, "/usr/local/PPU_SDK", exists) + require.Len(t, envs, 3) + assert.Equal(t, "CUDA_VISIBLE_DEVICES", envs[0].Key) + assert.Equal(t, "0,3", envs[0].Value) + assert.Equal(t, "NVIDIA_VISIBLE_DEVICES", envs[1].Key) + assert.Equal(t, "0,3", envs[1].Value) + assert.Equal(t, "LD_LIBRARY_PATH", envs[2].Key) + assert.Equal(t, "/usr/local/PPU_SDK/lib", envs[2].Value) + require.Len(t, mounts, 1) + assert.Equal(t, "/usr/local/PPU_SDK", mounts[0].HostPath) + assert.True(t, mounts[0].Readonly) + + envs, mounts = buildTHeadPpuExtraConfigures(nil, "/usr/local/PPU_SDK", exists) + assert.Nil(t, envs) + assert.Nil(t, mounts) +} + +func TestParseTHeadPpuNodeIndex(t *testing.T) { + idx, ok := parseTHeadPpuNodeIndex("alixpu_ppu3") + assert.True(t, ok) + assert.Equal(t, 3, idx) + _, ok = parseTHeadPpuNodeIndex("alixpu") + assert.False(t, ok) +} + +func TestTHeadPpuSmiPathDefault(t *testing.T) { + orig := options.HostOptions.THeadPpuSmiPath + defer func() { options.HostOptions.THeadPpuSmiPath = orig }() + options.HostOptions.THeadPpuSmiPath = "" + assert.Equal(t, "/usr/local/bin/ppu-smi", tHeadPpuSmiPath()) + options.HostOptions.THeadPpuSmiPath = "/opt/bin/ppu-smi" + assert.Equal(t, "/opt/bin/ppu-smi", tHeadPpuSmiPath()) +} diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index a4f667a9d9..253659f5b1 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -324,6 +324,81 @@ func (man *isolatedDeviceManager) probeContainerHygonDCUs(enable, enableHami boo log.Infof("==== hygon dcu probe finished: total %d devices", len(devs)) } +func (man *isolatedDeviceManager) probeContainerIluvatarGPUs(enable bool) { + if !enable { + log.Infof("iluvatar gpu probe skipped: enable_container_iluvatar_gpu=false") + return + } + devman, err := GetContainerDeviceManager(ContainerDeviceTypeIluvatarGpu) + if err != nil { + log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeIluvatarGpu, err) + return + } + devs, err := devman.ProbeDevices() + if err != nil { + log.Warningf("Probe container iluvatar gpu devices: %v", err) + return + } + if len(devs) == 0 { + log.Infof("iluvatar gpu probe finished: no devices found") + return + } + for idx, dev := range devs { + man.devices = append(man.devices, dev) + log.Infof("Add Container iluvatar GPU device: %d => %#v", idx, dev) + } +} + +func (man *isolatedDeviceManager) probeContainerTHeadPPUs(enable bool) { + if !enable { + log.Infof("t-head ppu probe skipped: enable_container_t_head_ppu=false") + return + } + devman, err := GetContainerDeviceManager(ContainerDeviceTypeTHeadPpu) + if err != nil { + log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeTHeadPpu, err) + return + } + devs, err := devman.ProbeDevices() + if err != nil { + log.Warningf("Probe container t-head ppu devices: %v", err) + return + } + if len(devs) == 0 { + log.Infof("t-head ppu probe finished: no devices found") + return + } + for idx, dev := range devs { + man.devices = append(man.devices, dev) + log.Infof("Add Container t-head PPU device: %d => %#v", idx, dev) + } +} + +func (man *isolatedDeviceManager) probeContainerKunlunxinXPUs(enable bool) { + if !enable { + log.Infof("kunlunxin xpu probe skipped: enable_container_kunlunxin_xpu=false") + return + } + devman, err := GetContainerDeviceManager(ContainerDeviceTypeKunlunxinXpu) + if err != nil { + log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeKunlunxinXpu, err) + return + } + devs, err := devman.ProbeDevices() + if err != nil { + log.Warningf("Probe container kunlunxin xpu devices: %v", err) + return + } + if len(devs) == 0 { + log.Infof("kunlunxin xpu probe finished: no devices found") + return + } + for idx, dev := range devs { + man.devices = append(man.devices, dev) + log.Infof("Add Container kunlunxin XPU device: %d => %#v", idx, dev) + } +} + func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) { if skipGPUs { return @@ -498,6 +573,9 @@ type SIsolatedDeviceProbeOptions struct { EnableContainerAscendNpuHAMI bool EnableContainerHygonDCU bool EnableContainerHygonDCUHAMI bool + EnableContainerIluvatarGPU bool + EnableContainerTHeadPPU bool + EnableContainerKunlunxinXPU bool EnableWhitelist bool SriovNics, OvsOffloadNics []HostNic @@ -515,6 +593,9 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps) man.probeContainerAscendNPUs(opts.EnableContainerAscendNpu, opts.EnableContainerAscendNpuHAMI) man.probeContainerHygonDCUs(opts.EnableContainerHygonDCU, opts.EnableContainerHygonDCUHAMI) + man.probeContainerIluvatarGPUs(opts.EnableContainerIluvatarGPU) + man.probeContainerTHeadPPUs(opts.EnableContainerTHeadPPU) + man.probeContainerKunlunxinXPUs(opts.EnableContainerKunlunxinXPU) } else { log.Infof("==== ProbePCIDevices: not container host, hygon container probe will NOT run (use host_type=container for hygon dcu)") devModels, err := man.getCustomIsolatedDeviceModels() diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 537571a30d..66614d27d7 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -288,6 +288,18 @@ type SHostOptions struct { HygonVdevConfDir string `help:"hygon vdcu config directory" default:"/etc/vdev"` HygonVgpuCacheDir string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"` + EnableContainerIluvatarGPU bool `help:"enable container iluvatar gpu" default:"true"` + IluvatarCorexHome string `help:"iluvatar corex home" default:"/usr/local/corex-4.4.0"` + IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/corex-4.4.0/bin/ixsmi"` + + EnableContainerTHeadPPU bool `help:"enable container t-head ppu" default:"true"` + THeadPpuSdkHome string `help:"t-head ppu sdk home" default:"/usr/local/PPU_SDK"` + THeadPpuSmiPath string `help:"t-head ppu-smi path" default:"/usr/local/bin/ppu-smi"` + + EnableContainerKunlunxinXPU bool `help:"enable container kunlunxin xpu" default:"true"` + KunlunxinXreHome string `help:"kunlunxin xre home" default:"/usr/local/xpu"` + KunlunxinXpuSmiPath string `help:"kunlunxin xpu-smi path" default:"/usr/local/bin/xpu-smi"` + EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` DisableReconcileContainer bool `help:"disable reconcile container" default:"false"` diff --git a/pkg/hostman/system_service/telegraf.go b/pkg/hostman/system_service/telegraf.go index e5c1f3c0d6..b7b0b3da3b 100644 --- a/pkg/hostman/system_service/telegraf.go +++ b/pkg/hostman/system_service/telegraf.go @@ -38,8 +38,11 @@ const ( TELEGRAF_INPUT_NETDEV = "ni_rsrc_mon" TELEGRAF_INPUT_VASMI = "vasmi" TELEGRAF_INPUT_HYSMI = "hysmi" + TELEGRAF_INPUT_IXSMI = "ixsmi" + TELEGRAF_INPUT_PPUSMI = "ppusmi" TELEGRAF_INPUT_NVIDIASMI = "nvidia-smi" TELEGRAF_INPUT_NPUSMI = "npu-smi" + TELEGRAF_INPUT_CONF_LIB_PATH = "lib_path" ) type STelegraf struct { @@ -346,6 +349,26 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string { conf += "\n" } + if ixsmi, ok := kwargs[TELEGRAF_INPUT_IXSMI]; ok { + ixsmiMap, _ := ixsmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_IXSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", ixsmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + if libPath, _ := ixsmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" { + conf += fmt.Sprintf(" lib_path = \"%s\"\n", libPath) + } + conf += "\n" + } + + if ppusmi, ok := kwargs[TELEGRAF_INPUT_PPUSMI]; ok { + ppusmiMap, _ := ppusmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_PPUSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", ppusmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + if libPath, _ := ppusmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" { + conf += fmt.Sprintf(" lib_path = \"%s\"\n", libPath) + } + conf += "\n" + } + if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok { conf += "[[inputs.nvidia_smi]]\n" conf += "\n" diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index d03865c465..94e1b5bf97 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -430,6 +430,9 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo modelPath := models.PickContainerModelMountPath(models.CollectContainerModelMountPaths(llm, sku), preferred) hasMountedModels := modelPath != "" || len(postOverlays) > 0 || models.SkuHasLocalHostPathModel(sku) hygon := models.HasHygonDevices(llm, sku) + iluvatar := models.HasIluvatarDevices(llm, sku) + thead := models.HasTHeadDevices(llm, sku) + kunlunxin := models.HasKunlunxinDevices(llm, sku) startScript := buildVLLMEntrypointScript(modelPath, tensorParallelSize, backendParameters, effSpec, hygon) envs := []*commonapi.ContainerKeyValue{ { @@ -462,7 +465,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo Envs: envs, }, } - if hygon { + if hygon || iluvatar || thead || kunlunxin { spec.Command = []string{"/bin/bash", "-c"} } if hasMountedModels { diff --git a/pkg/llm/drivers/llm_container/vllm_hygon_test.go b/pkg/llm/drivers/llm_container/vllm_hygon_test.go index 46db16c2f1..ba0101360f 100644 --- a/pkg/llm/drivers/llm_container/vllm_hygon_test.go +++ b/pkg/llm/drivers/llm_container/vllm_hygon_test.go @@ -82,3 +82,111 @@ func TestVLLMGetContainerSpecNvidiaNoHygonRuntime(t *testing.T) { t.Fatalf("args should not source dtk env for nvidia, got %#v", spec.Args) } } + +func TestVLLMGetContainerSpecIluvatarRuntime(t *testing.T) { + v := newVLLM().(*vllm) + hostPaths := api.HostPaths{ + { + Type: "directory", + Path: "/data/model/Qwen3.5-122B-A10B-w4a8", + Containers: api.ContainerHostPathRelations{ + "0": {MountPath: "/data/model/Qwen3.5-122B-A10B-w4a8"}, + }, + }, + } + sku := &models.SLLMSku{ + SLLMSkuBase: models.SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + }, + }, + LLMType: string(api.LLM_CONTAINER_VLLM), + Source: api.LLM_MODEL_SOURCE_LOCAL_PATH, + LocalPath: "/data/model/Qwen3.5-122B-A10B-w4a8", + } + sku.HostPaths = &hostPaths + image := &models.SLLMImage{} + out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "") + if out == nil { + t.Fatal("expected container spec") + } + spec := &out.ContainerSpec + if spec.Capabilities != nil { + t.Fatalf("expected nil capabilities for iluvatar, got %#v", spec.Capabilities) + } + if spec.SecurityContext != nil { + t.Fatalf("expected nil security context for iluvatar, got %#v", spec.SecurityContext) + } + if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" { + t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command) + } + if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") { + t.Fatalf("args should not source dtk env for iluvatar, got %#v", spec.Args) + } + if len(spec.Args) == 0 || !strings.Contains(spec.Args[0], "--tensor-parallel-size 4") { + t.Fatalf("args = %#v, want tensor-parallel-size 4", spec.Args) + } +} + +func TestVLLMGetContainerSpecTHeadRuntime(t *testing.T) { + v := newVLLM().(*vllm) + sku := &models.SLLMSku{ + SLLMSkuBase: models.SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_THEAD_PPU}, + {DevType: computeapi.CONTAINER_DEV_THEAD_PPU}, + }, + }, + } + image := &models.SLLMImage{} + out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "") + if out == nil { + t.Fatal("expected container spec") + } + spec := &out.ContainerSpec + if spec.Capabilities != nil { + t.Fatalf("expected nil capabilities for t-head, got %#v", spec.Capabilities) + } + if spec.SecurityContext != nil { + t.Fatalf("expected nil security context for t-head, got %#v", spec.SecurityContext) + } + if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" { + t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command) + } + if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") { + t.Fatalf("args should not source dtk env for t-head, got %#v", spec.Args) + } +} + +func TestVLLMGetContainerSpecKunlunxinRuntime(t *testing.T) { + v := newVLLM().(*vllm) + sku := &models.SLLMSku{ + SLLMSkuBase: models.SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU}, + {DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU}, + }, + }, + } + image := &models.SLLMImage{} + out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "") + if out == nil { + t.Fatal("expected container spec") + } + spec := &out.ContainerSpec + if spec.Capabilities != nil { + t.Fatalf("expected nil capabilities for kunlunxin, got %#v", spec.Capabilities) + } + if spec.SecurityContext != nil { + t.Fatalf("expected nil security context for kunlunxin, got %#v", spec.SecurityContext) + } + if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" { + t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command) + } + if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") { + t.Fatalf("args should not source dtk env for kunlunxin, got %#v", spec.Args) + } +} diff --git a/pkg/llm/models/llm_base.go b/pkg/llm/models/llm_base.go index 821e000ebe..1de2596a63 100644 --- a/pkg/llm/models/llm_base.go +++ b/pkg/llm/models/llm_base.go @@ -403,6 +403,57 @@ func HasHygonDevices(llm *SLLM, sku *SLLMSku) bool { return false } +// HasIluvatarDevices reports whether effective devices include Iluvatar GPU. +func HasIluvatarDevices(llm *SLLM, sku *SLLMSku) bool { + devs := GetEffectiveDevices(llm, sku) + if devs == nil { + return false + } + for _, d := range *devs { + if strings.EqualFold(d.Vendor, "ILUVATAR") { + return true + } + if d.DevType == computeapi.CONTAINER_DEV_ILUVATAR_GPU { + return true + } + } + return false +} + +// HasTHeadDevices reports whether effective devices include T-Head PPU. +func HasTHeadDevices(llm *SLLM, sku *SLLMSku) bool { + devs := GetEffectiveDevices(llm, sku) + if devs == nil { + return false + } + for _, d := range *devs { + if strings.EqualFold(d.Vendor, "THEAD") { + return true + } + if d.DevType == computeapi.CONTAINER_DEV_THEAD_PPU { + return true + } + } + return false +} + +// HasKunlunxinDevices reports whether effective devices include Kunlunxin XPU. +func HasKunlunxinDevices(llm *SLLM, sku *SLLMSku) bool { + devs := GetEffectiveDevices(llm, sku) + if devs == nil { + return false + } + for _, d := range *devs { + if strings.EqualFold(d.Vendor, "KUNLUNXIN") { + return true + } + if d.DevType == computeapi.CONTAINER_DEV_KUNLUNXIN_XPU { + return true + } + } + return false +} + // GetEffectiveHostPaths returns the host_paths to apply with llm's override taking priority over sku. func GetEffectiveHostPaths(llm *SLLM, sku *SLLMSku) *api.HostPaths { var llmBase *SLLMBase diff --git a/pkg/llm/models/llm_devices_test.go b/pkg/llm/models/llm_devices_test.go index ebce8636da..133798ce46 100644 --- a/pkg/llm/models/llm_devices_test.go +++ b/pkg/llm/models/llm_devices_test.go @@ -63,3 +63,141 @@ func TestHasHygonDevices(t *testing.T) { t.Fatal("expected normalized GPU+HYGON vendor sku to be detected as Hygon") } } + +func TestHasIluvatarDevices(t *testing.T) { + iluvatarSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + }, + }, + } + if !HasIluvatarDevices(nil, iluvatarSku) { + t.Fatal("expected Iluvatar GPU sku to be detected") + } + + nvSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU}, + }, + }, + } + if HasIluvatarDevices(nil, nvSku) { + t.Fatal("expected NVIDIA sku not to be detected as Iluvatar") + } + + llm := &SLLM{ + SLLMBase: SLLMBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + }, + }, + } + if !HasIluvatarDevices(llm, nvSku) { + t.Fatal("expected llm device override to win over sku") + } + + normalizedSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.GPU_TYPE, Vendor: "ILUVATAR", Model: "BI-V150S"}, + }, + }, + } + if !HasIluvatarDevices(nil, normalizedSku) { + t.Fatal("expected normalized GPU+ILUVATAR vendor sku to be detected as Iluvatar") + } +} + +func TestHasTHeadDevices(t *testing.T) { + theadSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_THEAD_PPU}, + }, + }, + } + if !HasTHeadDevices(nil, theadSku) { + t.Fatal("expected T-Head PPU sku to be detected") + } + + nvSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU}, + }, + }, + } + if HasTHeadDevices(nil, nvSku) { + t.Fatal("expected NVIDIA sku not to be detected as T-Head") + } + + llm := &SLLM{ + SLLMBase: SLLMBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_THEAD_PPU}, + }, + }, + } + if !HasTHeadDevices(llm, nvSku) { + t.Fatal("expected llm device override to win over sku") + } + + normalizedSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.GPU_TYPE, Vendor: "THEAD", Model: "PPU-ZW810E"}, + }, + }, + } + if !HasTHeadDevices(nil, normalizedSku) { + t.Fatal("expected normalized GPU+THEAD vendor sku to be detected as T-Head") + } +} + +func TestHasKunlunxinDevices(t *testing.T) { + kunlunSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU}, + }, + }, + } + if !HasKunlunxinDevices(nil, kunlunSku) { + t.Fatal("expected Kunlunxin XPU sku to be detected") + } + + nvSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU}, + }, + }, + } + if HasKunlunxinDevices(nil, nvSku) { + t.Fatal("expected NVIDIA sku not to be detected as Kunlunxin") + } + + llm := &SLLM{ + SLLMBase: SLLMBase{ + Devices: &api.Devices{ + {DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU}, + }, + }, + } + if !HasKunlunxinDevices(llm, nvSku) { + t.Fatal("expected llm device override to win over sku") + } + + normalizedSku := &SLLMSku{ + SLLMSkuBase: SLLMSkuBase{ + Devices: &api.Devices{ + {DevType: computeapi.GPU_TYPE, Vendor: "KUNLUNXIN", Model: "P800 OAM"}, + }, + }, + } + if !HasKunlunxinDevices(nil, normalizedSku) { + t.Fatal("expected normalized GPU+KUNLUNXIN vendor sku to be detected as Kunlunxin") + } +} diff --git a/pkg/llm/models/sku.go b/pkg/llm/models/sku.go index 7f4000711b..58ca568714 100644 --- a/pkg/llm/models/sku.go +++ b/pkg/llm/models/sku.go @@ -86,7 +86,7 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred return input, nil } -// normalizeLLMSkuDevices maps legacy NVIDIA_* / HYGON_* / ASCEND_* DevTypes onto +// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* DevTypes onto // GPU|NPU + SharingMode, and defaults empty DevType/SharingMode appropriately. func normalizeLLMSkuDevices(devices *api.Devices) error { if devices == nil || len(*devices) == 0 { @@ -160,6 +160,21 @@ func normalizeLLMSkuDevice(dev *api.Device) { if dev.SharingMode == "" { dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI } + case computeapi.CONTAINER_DEV_ILUVATAR_GPU: + dev.DevType = computeapi.GPU_TYPE + if dev.SharingMode == "" { + dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE + } + case computeapi.CONTAINER_DEV_THEAD_PPU: + dev.DevType = computeapi.GPU_TYPE + if dev.SharingMode == "" { + dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE + } + case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU: + dev.DevType = computeapi.GPU_TYPE + if dev.SharingMode == "" { + dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE + } } if dev.SharingMode == "" { dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI @@ -173,6 +188,12 @@ func normalizeLLMSkuDevice(dev *api.Device) { dev.Vendor = "NVIDIA" case computeapi.CONTAINER_DEV_ASCEND_NPU, computeapi.CONTAINER_DEV_ASCEND_NPU_HAMI: dev.Vendor = "ASCEND" + case computeapi.CONTAINER_DEV_ILUVATAR_GPU: + dev.Vendor = "ILUVATAR" + case computeapi.CONTAINER_DEV_THEAD_PPU: + dev.Vendor = "THEAD" + case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU: + dev.Vendor = "KUNLUNXIN" } } dev.Vendor = canonicalizeLLMDeviceVendor(dev.Vendor) diff --git a/pkg/llm/models/sku_hami_test.go b/pkg/llm/models/sku_hami_test.go index 5840fedcd5..491b6703f3 100644 --- a/pkg/llm/models/sku_hami_test.go +++ b/pkg/llm/models/sku_hami_test.go @@ -83,6 +83,34 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) { wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI, wantVendor: "ASCEND", }, + { + name: "ILUVATAR_GPU", + in: api.Device{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, + wantVendor: "ILUVATAR", + }, + { + name: "THEAD_PPU", + in: api.Device{DevType: computeapi.CONTAINER_DEV_THEAD_PPU}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, + wantVendor: "THEAD", + }, + { + name: "KUNLUNXIN_XPU", + in: api.Device{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, + wantVendor: "KUNLUNXIN", + }, + { + name: "ILUVATAR vendor preserved and canonicalized", + in: api.Device{DevType: computeapi.GPU_TYPE, Vendor: "iluvatar", Model: "BI-V150S"}, + wantDevType: computeapi.GPU_TYPE, + wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI, + wantVendor: "ILUVATAR", + }, { name: "ASCEND vendor empty DevType defaults to NPU", in: api.Device{Vendor: "ASCEND", Model: "910B2"}, diff --git a/pkg/monitor/dbinit/measurements/all.go b/pkg/monitor/dbinit/measurements/all.go index 5e08797867..2b0dd5cf46 100644 --- a/pkg/monitor/dbinit/measurements/all.go +++ b/pkg/monitor/dbinit/measurements/all.go @@ -32,6 +32,8 @@ var All = []SMeasurement{ vasmi, npuSmi, hysmi, + ixsmi, + ppusmi, worker, serviceHttpCode, diff --git a/pkg/monitor/dbinit/measurements/ixsmi.go b/pkg/monitor/dbinit/measurements/ixsmi.go new file mode 100644 index 0000000000..c7856dff74 --- /dev/null +++ b/pkg/monitor/dbinit/measurements/ixsmi.go @@ -0,0 +1,58 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var ixsmi = SMeasurement{ + Context: []SMonitorContext{ + { + "ixsmi", "Iluvatar GPU metrics", + monitor.METRIC_RES_TYPE_HOST, monitor.METRIC_DATABASE_TELE, + }, + }, + Metrics: []SMetric{ + { + "temperature_gpu", "GPU temperature", "", + }, + { + "temperature_memory", "GPU memory temperature", "", + }, + { + "memory_total", "GPU memory total size", "", + }, + { + "memory_free", "GPU memory free size", "", + }, + { + "memory_used", "GPU memory used size", "", + }, + { + "utilization_gpu", "GPU utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "utilization_memory", "GPU memory utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "power_draw", "GPU power draw", "", + }, + { + "clocks_current_sm", "GPU current SM clocks, MHz", "", + }, + { + "clocks_current_memory", "GPU current memory clocks, MHz", "", + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/metrics.csv b/pkg/monitor/dbinit/measurements/metrics.csv index 51df79dd4a..49639b7805 100644 --- a/pkg/monitor/dbinit/measurements/metrics.csv +++ b/pkg/monitor/dbinit/measurements/metrics.csv @@ -264,6 +264,26 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit "hysmi","Hygon DCU metrics","host","telegraf","utilization_memory","DCU memory utilization","%" "hysmi","Hygon DCU metrics","host","telegraf","utilization_encoder","DCU encoder utilization","%" "hysmi","Hygon DCU metrics","host","telegraf","utilization_decoder","DCU decoder utilization","%" +"ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_gpu","GPU temperature","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_memory","GPU memory temperature","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_total","GPU memory total size","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_free","GPU memory free size","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_used","GPU memory used size","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%" +"ixsmi","Iluvatar GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%" +"ixsmi","Iluvatar GPU metrics","host","telegraf","power_draw","GPU power draw","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_sm","GPU current SM clocks, MHz","" +"ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz","" +"ppusmi","T-Head PPU metrics","host","telegraf","temperature_gpu","PPU temperature","" +"ppusmi","T-Head PPU metrics","host","telegraf","temperature_memory","PPU memory temperature","" +"ppusmi","T-Head PPU metrics","host","telegraf","memory_total","PPU memory total size","" +"ppusmi","T-Head PPU metrics","host","telegraf","memory_free","PPU memory free size","" +"ppusmi","T-Head PPU metrics","host","telegraf","memory_used","PPU memory used size","" +"ppusmi","T-Head PPU metrics","host","telegraf","utilization_gpu","PPU utilization","%" +"ppusmi","T-Head PPU metrics","host","telegraf","utilization_memory","PPU memory utilization","%" +"ppusmi","T-Head PPU metrics","host","telegraf","power_draw","PPU power draw","" +"ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_sm","PPU current CU clocks, MHz","" +"ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_memory","PPU current memory clocks, MHz","" "worker","Worker queue","system","system","active_worker_cnt","Active Worker Count","NULL" "worker","Worker queue","system","system","max_worker_count","Max Worker Count","NULL" "worker","Worker queue","system","system","detach_worker_cnt","Detach worker Count","NULL" diff --git a/pkg/monitor/dbinit/measurements/ppusmi.go b/pkg/monitor/dbinit/measurements/ppusmi.go new file mode 100644 index 0000000000..27f48290d8 --- /dev/null +++ b/pkg/monitor/dbinit/measurements/ppusmi.go @@ -0,0 +1,58 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package measurements + +import "yunion.io/x/onecloud/pkg/apis/monitor" + +var ppusmi = SMeasurement{ + Context: []SMonitorContext{ + { + "ppusmi", "T-Head PPU metrics", + monitor.METRIC_RES_TYPE_HOST, monitor.METRIC_DATABASE_TELE, + }, + }, + Metrics: []SMetric{ + { + "temperature_gpu", "PPU temperature", "", + }, + { + "temperature_memory", "PPU memory temperature", "", + }, + { + "memory_total", "PPU memory total size", "", + }, + { + "memory_free", "PPU memory free size", "", + }, + { + "memory_used", "PPU memory used size", "", + }, + { + "utilization_gpu", "PPU utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "utilization_memory", "PPU memory utilization", monitor.METRIC_UNIT_PERCENT, + }, + { + "power_draw", "PPU power draw", "", + }, + { + "clocks_current_sm", "PPU current CU clocks, MHz", "", + }, + { + "clocks_current_memory", "PPU current memory clocks, MHz", "", + }, + }, +}