feat(host,llm): add Iluvatar GPU container device support (#25464)

* feat(host,llm): add Iluvatar GPU container device support

* feat(host,llm,monitor): add Thead PPU container device support

* feat(host,llm): add Kunlunxin XPU container device support

pkg/apis/compute/container.go

@@ -46,6 +46,9 @@ const (
 	CONTAINER_DEV_VASTAITECH_GPU   = "VASTAITECH_GPU"
 	CONTAINER_DEV_HYGON_DCU        = "HYGON_DCU"
 	CONTAINER_DEV_HYGON_DCU_HAMI   = "HYGON_DCU_HAMI"
+	CONTAINER_DEV_ILUVATAR_GPU     = "ILUVATAR_GPU"
+	CONTAINER_DEV_THEAD_PPU        = "THEAD_PPU"
+	CONTAINER_DEV_KUNLUNXIN_XPU    = "KUNLUNXIN_XPU"
 )
 
 var (
@@ -58,6 +61,9 @@ var (
 		CONTAINER_DEV_VASTAITECH_GPU,
 		CONTAINER_DEV_HYGON_DCU,
 		CONTAINER_DEV_HYGON_DCU_HAMI,
+		CONTAINER_DEV_ILUVATAR_GPU,
+		CONTAINER_DEV_THEAD_PPU,
+		CONTAINER_DEV_KUNLUNXIN_XPU,
 	}
 )
 

pkg/apis/compute/isolated_device_const.go

@@ -50,6 +50,9 @@ const (
 	VASTAITECH_VENDOR_ID = "1ec6"
 	HYGON_VENDOR_ID      = "1d94"
 	ASCEND_VENDOR_ID     = "19e5"
+	ILUVATAR_VENDOR_ID   = "1e3e"
+	THEAD_VENDOR_ID      = "1ded"
+	KUNLUNXIN_VENDOR_ID  = "1d22"
 )
 
 var ID_VENDOR_MAP = map[string]string{
@@ -58,6 +61,9 @@ var ID_VENDOR_MAP = map[string]string{
 	VASTAITECH_VENDOR_ID: "VASTAITECH",
 	ASCEND_VENDOR_ID:     "ASCEND",
 	HYGON_VENDOR_ID:      "HYGON",
+	ILUVATAR_VENDOR_ID:   "ILUVATAR",
+	THEAD_VENDOR_ID:      "THEAD",
+	KUNLUNXIN_VENDOR_ID:  "KUNLUNXIN",
 }
 
 var VENDOR_ID_MAP = map[string]string{
@@ -66,6 +72,9 @@ var VENDOR_ID_MAP = map[string]string{
 	"VASTAITECH": VASTAITECH_VENDOR_ID,
 	"ASCEND":     ASCEND_VENDOR_ID,
 	"HYGON":      HYGON_VENDOR_ID,
+	"ILUVATAR":   ILUVATAR_VENDOR_ID,
+	"THEAD":      THEAD_VENDOR_ID,
+	"KUNLUNXIN":  KUNLUNXIN_VENDOR_ID,
 }
 
 const (
@@ -91,6 +100,9 @@ var GPU_TYPES = []string{
 	CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE,
 	CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_VASTAITECH_GPU,
 	CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI,
+	CONTAINER_DEV_ILUVATAR_GPU,
+	CONTAINER_DEV_THEAD_PPU,
+	CONTAINER_DEV_KUNLUNXIN_XPU,
 }
 
 var NETINT_TYPES = []string{
@@ -102,6 +114,9 @@ var VALID_CONTAINER_DEVICE_TYPES = []string{
 	CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI,
 	CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU,
 	CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI,
+	CONTAINER_DEV_ILUVATAR_GPU,
+	CONTAINER_DEV_THEAD_PPU,
+	CONTAINER_DEV_KUNLUNXIN_XPU,
 }
 
 var VALID_PASSTHROUGH_TYPES = []string{
@@ -122,6 +137,9 @@ var VITRUAL_DEVICE_TYPES = []string{
 	CONTAINER_DEV_VASTAITECH_GPU,
 	CONTAINER_DEV_HYGON_DCU,
 	CONTAINER_DEV_HYGON_DCU_HAMI,
+	CONTAINER_DEV_ILUVATAR_GPU,
+	CONTAINER_DEV_THEAD_PPU,
+	CONTAINER_DEV_KUNLUNXIN_XPU,
 }
 
 func init() {

pkg/hostman/hostinfo/hostinfo.go

@@ -2362,6 +2362,9 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		EnableContainerAscendNpuHAMI: options.HostOptions.EnableContainerAscendNPUHami,
 		EnableContainerHygonDCU:      options.HostOptions.EnableContainerHygonDCU,
 		EnableContainerHygonDCUHAMI:  options.HostOptions.EnableContainerHygonDCUHami,
+		EnableContainerIluvatarGPU:   options.HostOptions.EnableContainerIluvatarGPU,
+		EnableContainerTHeadPPU:      options.HostOptions.EnableContainerTHeadPPU,
+		EnableContainerKunlunxinXPU:  options.HostOptions.EnableContainerKunlunxinXPU,
 		EnableWhitelist:              options.HostOptions.EnableIsolatedDeviceWhitelist,
 		SriovNics:                    sriovNics,
 		OvsOffloadNics:               offloadNics,
@@ -2691,6 +2694,8 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 	hasNetint := false
 	hasVasmi := false
 	hasHygon := false
+	hasIluvatar := false
+	hasTHead := false
 	hasNvidiasmi := false
 	hasNpusmi := false
 	for _, dev := range devs {
@@ -2698,6 +2703,12 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 		if vendorId == api.HYGON_VENDOR_ID {
 			hasHygon = true
 		}
+		if vendorId == api.ILUVATAR_VENDOR_ID {
+			hasIluvatar = true
+		}
+		if vendorId == api.THEAD_VENDOR_ID {
+			hasTHead = true
+		}
 		if !utils.IsInStringArray(dev.GetSharingMode(), api.VIRTUAL_SHARING_MODES) {
 			continue
 		}
@@ -2744,6 +2755,38 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: options.HostOptions.HygonHySmiPath,
 		}
 	}
+	if hasIluvatar {
+		corexHome := options.HostOptions.IluvatarCorexHome
+		if corexHome == "" {
+			corexHome = "/usr/local/corex-4.4.0"
+		}
+		ixsmiPath := options.HostOptions.IluvatarIxsmiPath
+		if ixsmiPath == "" {
+			ixsmiPath = path.Join(corexHome, "bin", "ixsmi")
+		}
+		conf[system_service.TELEGRAF_INPUT_IXSMI] = map[string]interface{}{
+			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: ixsmiPath,
+			system_service.TELEGRAF_INPUT_CONF_LIB_PATH: path.Join(corexHome, "lib64"),
+		}
+	}
+	if hasTHead {
+		sdkHome := options.HostOptions.THeadPpuSdkHome
+		if sdkHome == "" {
+			sdkHome = "/usr/local/PPU_SDK"
+		}
+		libPath := path.Join(sdkHome, "lib64")
+		if !fileutils2.Exists(libPath) && fileutils2.Exists(path.Join(sdkHome, "lib")) {
+			libPath = path.Join(sdkHome, "lib")
+		}
+		smiPath := options.HostOptions.THeadPpuSmiPath
+		if smiPath == "" {
+			smiPath = "/usr/local/bin/ppu-smi"
+		}
+		conf[system_service.TELEGRAF_INPUT_PPUSMI] = map[string]interface{}{
+			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: smiPath,
+			system_service.TELEGRAF_INPUT_CONF_LIB_PATH: libPath,
+		}
+	}
 	if hasNvidiasmi {
 		conf[system_service.TELEGRAF_INPUT_NVIDIASMI] = struct{}{}
 	}

pkg/hostman/isolated_device/container_device.go

@@ -50,6 +50,9 @@ const (
 	ContainerDeviceTypeVastaitechGpu  ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU
 	ContainerDeviceTypeHygonDcu       ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU
 	ContainerDeviceTypeHygonDcuHami   ContainerDeviceType = api.CONTAINER_DEV_HYGON_DCU_HAMI
+	ContainerDeviceTypeIluvatarGpu    ContainerDeviceType = api.CONTAINER_DEV_ILUVATAR_GPU
+	ContainerDeviceTypeTHeadPpu       ContainerDeviceType = api.CONTAINER_DEV_THEAD_PPU
+	ContainerDeviceTypeKunlunxinXpu   ContainerDeviceType = api.CONTAINER_DEV_KUNLUNXIN_XPU
 )
 
 func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) {

pkg/hostman/isolated_device/container_device/iluvatar_gpu.go

@@ -0,0 +1,266 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain the copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newIluvatarGPUManager())
+}
+
+type iluvatarGPUManager struct{}
+
+func newIluvatarGPUManager() *iluvatarGPUManager {
+	return &iluvatarGPUManager{}
+}
+
+func (m *iluvatarGPUManager) GetRegisterType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeIluvatarGpu
+}
+
+func (m *iluvatarGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return getIluvatarGPUs(m)
+}
+
+func (m *iluvatarGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *iluvatarGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	if dev.IsolatedDevice == nil {
+		return nil, nil, errors.Errorf("isolated device is nil")
+	}
+	iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+	if iDev == nil {
+		return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id)
+	}
+	gpuDev, ok := iDev.(*iluvatarGPU)
+	if !ok {
+		return nil, nil, errors.Errorf("device %s is not iluvatar gpu", dev.IsolatedDevice.Id)
+	}
+	minor := gpuDev.GetDeviceMinor()
+	if minor < 0 {
+		minor = gpuDev.GetIndex()
+	}
+	ctrDevs := []*runtimeapi.Device{}
+	node := iluvatarDevNode(minor)
+	if hygonPathExists(node) {
+		ctrDevs = append(ctrDevs, iluvatarDeviceSpec(node))
+	} else {
+		log.Warningf("iluvatar container device %s not found, skip", node)
+	}
+	return ctrDevs, iluvatarCommonDevices(), nil
+}
+
+func (m *iluvatarGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	indices := collectIluvatarVisibleIndices(devs)
+	return buildIluvatarExtraConfigures(indices, iluvatarCorexHome(), hygonPathExists)
+}
+
+type iluvatarGPU struct {
+	manager isolated_device.IContainerDeviceManager
+	*BaseDevice
+
+	memSize     int
+	gpuIndex    int
+	deviceMinor int
+	uuid        string
+}
+
+func (dev *iluvatarGPU) GetMemorySize() int {
+	return dev.memSize
+}
+
+func (dev *iluvatarGPU) GetIndex() int {
+	return dev.gpuIndex
+}
+
+func (dev *iluvatarGPU) GetDeviceMinor() int {
+	return dev.deviceMinor
+}
+
+func (dev *iluvatarGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
+	return dev.manager
+}
+
+func iluvatarCommonDevices() []*runtimeapi.Device {
+	devs := []*runtimeapi.Device{}
+	for _, p := range collectIluvatarCommonDevicePaths(hygonPathExists) {
+		devs = append(devs, iluvatarDeviceSpec(p))
+	}
+	return devs
+}
+
+func iluvatarCorexHome() string {
+	home := options.HostOptions.IluvatarCorexHome
+	if home == "" {
+		return defaultIluvatarCorexHome
+	}
+	return home
+}
+
+func iluvatarIxsmiPath() string {
+	p := options.HostOptions.IluvatarIxsmiPath
+	if p != "" {
+		return p
+	}
+	return path.Join(iluvatarCorexHome(), "bin", "ixsmi")
+}
+
+func iluvatarLDLibraryPath() string {
+	lib64 := path.Join(iluvatarCorexHome(), "lib64")
+	existing := os.Getenv("LD_LIBRARY_PATH")
+	if existing == "" {
+		return lib64
+	}
+	return existing + ":" + lib64
+}
+
+func runIxsmi(args ...string) (string, error) {
+	ixsmiPath := iluvatarIxsmiPath()
+	cmd := procutils.NewRemoteCommandAsFarAsPossible(ixsmiPath, args...)
+	cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + iluvatarLDLibraryPath()})
+	out, err := cmd.Output()
+	if err != nil {
+		return string(out), errors.Wrapf(err, "ixsmi %s", strings.Join(args, " "))
+	}
+	return string(out), nil
+}
+
+func lookupIluvatarPCIDevice(busId, modelName string) *isolated_device.PCIDevice {
+	cands := iluvatarPCIAddrCandidates(busId)
+	for _, addr := range cands {
+		pciOutput, err := isolated_device.GetPCIStrByAddr(addr)
+		if err != nil || len(pciOutput) == 0 {
+			log.Warningf("iluvatar GetPCIStrByAddr %s: %v", addr, err)
+			continue
+		}
+		dev := isolated_device.NewPCIDevice2(pciOutput[0])
+		if modelName != "" {
+			dev.ModelName = modelName
+		}
+		return dev
+	}
+	fallbackAddr := busId
+	if len(cands) > 0 {
+		fallbackAddr = cands[len(cands)-1]
+	}
+	return &isolated_device.PCIDevice{
+		VendorId:   computeapi.ILUVATAR_VENDOR_ID,
+		VendorName: "ILUVATAR",
+		Addr:       fallbackAddr,
+		ModelName:  modelName,
+		DeviceName: modelName,
+	}
+}
+
+func collectIluvatarVisibleIndices(devs []*hostapi.ContainerDevice) []string {
+	indices := []string{}
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		devMan := iDev.GetContainerDeviceManager()
+		if _, ok := devMan.(*iluvatarGPUManager); !ok {
+			continue
+		}
+		if dev.IsolatedDevice.Path != "" {
+			indices = append(indices, dev.IsolatedDevice.Path)
+			continue
+		}
+		if dev.IsolatedDevice.Index >= 0 {
+			indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index))
+		}
+	}
+	return indices
+}
+
+func getIluvatarGPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) {
+	ixsmiPath := iluvatarIxsmiPath()
+	if !hygonPathExists(ixsmiPath) {
+		log.Infof("iluvatar gpu probe skipped: ixsmi not found at %s", ixsmiPath)
+		return nil, nil
+	}
+
+	tableOut, err := runIxsmi()
+	if err != nil {
+		return nil, err
+	}
+	parsed := parseIxsmiTable(tableOut)
+	if len(parsed) == 0 {
+		log.Infof("iluvatar gpu probe: no devices parsed from ixsmi table")
+		return nil, nil
+	}
+
+	listOut, err := runIxsmi("-L")
+	if err != nil {
+		log.Warningf("ixsmi -L failed: %v", err)
+	} else {
+		parsed = mergeIluvatarProbe(parsed, parseIxsmiList(listOut))
+	}
+
+	devs := make([]isolated_device.IDevice, 0, len(parsed))
+	for _, gpu := range parsed {
+		if gpu.ComputeMode != "" && gpu.ComputeMode != iluvatarComputeModeOK {
+			log.Warningf("iluvatar gpu %d compute mode %s, skip", gpu.Index, gpu.ComputeMode)
+			continue
+		}
+		pciDev := lookupIluvatarPCIDevice(gpu.BusId, gpu.Name)
+		indexStr := strconv.Itoa(gpu.Index)
+		minor := resolveIluvatarDeviceMinor(gpu.Index, hygonPathExists)
+		if minor < 0 {
+			log.Warningf("iluvatar gpu %d: %s not found, fallback to index for container device", gpu.Index, iluvatarDevNode(gpu.Index))
+		}
+		dev := &iluvatarGPU{
+			manager:     manager,
+			BaseDevice:  NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1),
+			memSize:     gpu.MemorySizeMB,
+			gpuIndex:    gpu.Index,
+			deviceMinor: minor,
+			uuid:        gpu.UUID,
+		}
+		if gpu.Name != "" {
+			dev.SetModelName(gpu.Name)
+		}
+		log.Infof("iluvatar gpu idx=%d minor=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s",
+			gpu.Index, minor, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, iluvatarDevNode(gpu.Index))
+		devs = append(devs, dev)
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	return devs, nil
+}

pkg/hostman/isolated_device/container_device/iluvatar_ixsmi.go

@@ -0,0 +1,244 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain the copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const (
+	defaultIluvatarCorexHome = "/usr/local/corex-4.4.0"
+	iluvatarCorexAlias       = "/usr/local/corex"
+	iluvatarCtlDevicePath    = "/dev/itrctl"
+	iluvatarComputeModeOK    = "Default"
+)
+
+type parsedIluvatarGPU struct {
+	Index        int
+	Name         string
+	BusId        string
+	MemorySizeMB int
+	ComputeMode  string
+	UUID         string
+}
+
+var (
+	ixsmiMemRe  = regexp.MustCompile(`(\d+)\s*MiB\s*/\s*(\d+)\s*MiB`)
+	ixsmiListRe = regexp.MustCompile(`(?i)^GPU\s+(\d+):\s+.+\(UUID:\s*([^)]+)\)`)
+)
+
+func splitIxsmiCols(line string) []string {
+	line = strings.TrimSpace(line)
+	line = strings.Trim(line, "|")
+	parts := strings.Split(line, "|")
+	cols := make([]string, 0, len(parts))
+	for _, p := range parts {
+		cols = append(cols, strings.TrimSpace(p))
+	}
+	return cols
+}
+
+func parseIxsmiTable(output string) []*parsedIluvatarGPU {
+	var out []*parsedIluvatarGPU
+	var cur *parsedIluvatarGPU
+	for _, raw := range strings.Split(output, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" || strings.HasPrefix(line, "+") || !strings.HasPrefix(line, "|") {
+			continue
+		}
+		cols := splitIxsmiCols(line)
+		if len(cols) < 3 {
+			continue
+		}
+		left := cols[0]
+		if strings.HasPrefix(left, "GPU") || strings.HasPrefix(left, "Fan") ||
+			strings.HasPrefix(left, "IX-ML") || strings.HasPrefix(left, "Processes") ||
+			strings.HasPrefix(left, "No running") {
+			continue
+		}
+		fields := strings.Fields(left)
+		if len(fields) >= 2 {
+			if idx, err := strconv.Atoi(fields[0]); err == nil {
+				name := strings.TrimSpace(strings.TrimPrefix(left, fields[0]))
+				cur = &parsedIluvatarGPU{
+					Index: idx,
+					Name:  name,
+					BusId: cols[1],
+				}
+				continue
+			}
+		}
+		if cur == nil {
+			continue
+		}
+		if m := ixsmiMemRe.FindStringSubmatch(cols[1]); len(m) == 3 {
+			cur.MemorySizeMB, _ = strconv.Atoi(m[2])
+		}
+		modeFields := strings.Fields(cols[len(cols)-1])
+		if len(modeFields) > 0 {
+			cur.ComputeMode = modeFields[len(modeFields)-1]
+		}
+		out = append(out, cur)
+		cur = nil
+	}
+	if cur != nil {
+		out = append(out, cur)
+	}
+	return out
+}
+
+func parseIxsmiList(output string) map[int]string {
+	ret := map[int]string{}
+	for _, raw := range strings.Split(output, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" {
+			continue
+		}
+		m := ixsmiListRe.FindStringSubmatch(line)
+		if len(m) != 3 {
+			continue
+		}
+		idx, err := strconv.Atoi(m[1])
+		if err != nil {
+			continue
+		}
+		ret[idx] = strings.TrimSpace(m[2])
+	}
+	return ret
+}
+
+func mergeIluvatarProbe(table []*parsedIluvatarGPU, uuids map[int]string) []*parsedIluvatarGPU {
+	for _, gpu := range table {
+		if gpu == nil {
+			continue
+		}
+		if uuid, ok := uuids[gpu.Index]; ok {
+			gpu.UUID = uuid
+		}
+	}
+	return table
+}
+
+func iluvatarPCIAddrCandidates(busId string) []string {
+	busId = strings.TrimSpace(busId)
+	if busId == "" {
+		return nil
+	}
+	cands := make([]string, 0, 3)
+	seen := map[string]bool{}
+	add := func(s string) {
+		if s == "" || seen[s] {
+			return
+		}
+		seen[s] = true
+		cands = append(cands, s)
+	}
+	add(busId)
+	parts := strings.Split(busId, ":")
+	if len(parts) == 3 {
+		domain, bus, fn := parts[0], parts[1], parts[2]
+		if len(domain) > 4 {
+			add(domain[len(domain)-4:] + ":" + bus + ":" + fn)
+		}
+		add(bus + ":" + fn)
+	}
+	return cands
+}
+
+func iluvatarDevNode(minor int) string {
+	return fmt.Sprintf("/dev/iluvatar%d", minor)
+}
+
+func resolveIluvatarDeviceMinor(index int, pathExists func(string) bool) int {
+	if pathExists != nil && pathExists(iluvatarDevNode(index)) {
+		return index
+	}
+	return -1
+}
+
+func iluvatarDeviceSpec(devPath string) *runtimeapi.Device {
+	return &runtimeapi.Device{
+		ContainerPath: devPath,
+		HostPath:      devPath,
+		Permissions:   "rwm",
+	}
+}
+
+func collectIluvatarCommonDevicePaths(pathExists func(string) bool) []string {
+	if pathExists != nil && pathExists(iluvatarCtlDevicePath) {
+		return []string{iluvatarCtlDevicePath}
+	}
+	return nil
+}
+
+func normalizeIluvatarCorexHome(corexHome string) string {
+	if corexHome == "" {
+		return defaultIluvatarCorexHome
+	}
+	return corexHome
+}
+
+func buildIluvatarRuntimeEnvs(indices []string, corexHome string) []*runtimeapi.KeyValue {
+	corexHome = normalizeIluvatarCorexHome(corexHome)
+	return []*runtimeapi.KeyValue{
+		{
+			Key:   "IX_VISIBLE_DEVICES",
+			Value: strings.Join(indices, ","),
+		},
+		{
+			Key:   "COREX_HOME",
+			Value: corexHome,
+		},
+		{
+			Key:   "LD_LIBRARY_PATH",
+			Value: path.Join(corexHome, "lib64"),
+		},
+	}
+}
+
+func buildIluvatarRuntimeMounts(corexHome string, pathExists func(string) bool) []*runtimeapi.Mount {
+	corexHome = normalizeIluvatarCorexHome(corexHome)
+	if pathExists == nil || !pathExists(corexHome) {
+		return nil
+	}
+	mounts := []*runtimeapi.Mount{
+		{
+			ContainerPath: corexHome,
+			HostPath:      corexHome,
+			Readonly:      true,
+		},
+	}
+	if path.Clean(corexHome) != path.Clean(iluvatarCorexAlias) {
+		mounts = append(mounts, &runtimeapi.Mount{
+			ContainerPath: iluvatarCorexAlias,
+			HostPath:      corexHome,
+			Readonly:      true,
+		})
+	}
+	return mounts
+}
+
+func buildIluvatarExtraConfigures(indices []string, corexHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	if len(indices) == 0 {
+		return nil, nil
+	}
+	return buildIluvatarRuntimeEnvs(indices, corexHome), buildIluvatarRuntimeMounts(corexHome, pathExists)
+}

pkg/hostman/isolated_device/container_device/iluvatar_ixsmi_test.go

@@ -0,0 +1,106 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain the copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIluvatarDevNode(t *testing.T) {
+	assert.Equal(t, "/dev/iluvatar0", iluvatarDevNode(0))
+	assert.Equal(t, "/dev/iluvatar7", iluvatarDevNode(7))
+}
+
+func TestResolveIluvatarDeviceMinor(t *testing.T) {
+	exists := func(p string) bool { return p == "/dev/iluvatar3" }
+	assert.Equal(t, 3, resolveIluvatarDeviceMinor(3, exists))
+	assert.Equal(t, -1, resolveIluvatarDeviceMinor(0, exists))
+	assert.Equal(t, -1, resolveIluvatarDeviceMinor(0, nil))
+}
+
+func TestCollectIluvatarCommonDevicePaths(t *testing.T) {
+	assert.Nil(t, collectIluvatarCommonDevicePaths(nil))
+	assert.Nil(t, collectIluvatarCommonDevicePaths(func(string) bool { return false }))
+	assert.Equal(t, []string{"/dev/itrctl"}, collectIluvatarCommonDevicePaths(func(p string) bool {
+		return p == "/dev/itrctl"
+	}))
+}
+
+func TestIluvatarDeviceSpec(t *testing.T) {
+	dev := iluvatarDeviceSpec("/dev/iluvatar2")
+	require.NotNil(t, dev)
+	assert.Equal(t, "/dev/iluvatar2", dev.HostPath)
+	assert.Equal(t, "/dev/iluvatar2", dev.ContainerPath)
+	assert.Equal(t, "rwm", dev.Permissions)
+}
+
+func TestBuildIluvatarExtraConfigures(t *testing.T) {
+	exists := func(p string) bool { return p == "/usr/local/corex-4.4.0" }
+	envs, mounts := buildIluvatarExtraConfigures([]string{"0", "1", "2", "3"}, "/usr/local/corex-4.4.0", exists)
+	require.Len(t, envs, 3)
+	assert.Equal(t, "IX_VISIBLE_DEVICES", envs[0].Key)
+	assert.Equal(t, "0,1,2,3", envs[0].Value)
+	assert.Equal(t, "COREX_HOME", envs[1].Key)
+	assert.Equal(t, "/usr/local/corex-4.4.0", envs[1].Value)
+	assert.Equal(t, "LD_LIBRARY_PATH", envs[2].Key)
+	assert.Equal(t, "/usr/local/corex-4.4.0/lib64", envs[2].Value)
+
+	require.Len(t, mounts, 2)
+	assert.Equal(t, "/usr/local/corex-4.4.0", mounts[0].HostPath)
+	assert.Equal(t, "/usr/local/corex-4.4.0", mounts[0].ContainerPath)
+	assert.True(t, mounts[0].Readonly)
+	assert.Equal(t, "/usr/local/corex-4.4.0", mounts[1].HostPath)
+	assert.Equal(t, "/usr/local/corex", mounts[1].ContainerPath)
+	assert.True(t, mounts[1].Readonly)
+
+	envs, mounts = buildIluvatarExtraConfigures(nil, "/usr/local/corex-4.4.0", exists)
+	assert.Nil(t, envs)
+	assert.Nil(t, mounts)
+}
+
+func TestBuildIluvatarRuntimeMountsAliasOnly(t *testing.T) {
+	exists := func(p string) bool { return p == "/usr/local/corex" }
+	mounts := buildIluvatarRuntimeMounts("/usr/local/corex", exists)
+	require.Len(t, mounts, 1)
+	assert.Equal(t, "/usr/local/corex", mounts[0].HostPath)
+	assert.Equal(t, "/usr/local/corex", mounts[0].ContainerPath)
+}
+
+func TestBuildIluvatarRuntimeMountsMissingHome(t *testing.T) {
+	assert.Nil(t, buildIluvatarRuntimeMounts("/usr/local/corex-4.4.0", func(string) bool { return false }))
+}
+
+func TestParseIxsmiTableTwoGpus(t *testing.T) {
+	input := `| 0    Iluvatar BI-V150S           | 00000000:26:00.0     | 500MHz    1600MHz    |
+| N/A  35C   P0       N/A / N/A    | 68MiB / 32768MiB     | 0%        Default    |
+| 1    Iluvatar BI-V150S           | 00000000:29:00.0     | 500MHz    1600MHz    |
+| N/A  33C   P0       53W / 450W   | 68MiB / 32768MiB     | 0%        Default    |`
+	gpus := parseIxsmiTable(input)
+	require.Len(t, gpus, 2)
+	assert.Equal(t, 0, gpus[0].Index)
+	assert.Equal(t, "Iluvatar BI-V150S", gpus[0].Name)
+	assert.Equal(t, "00000000:26:00.0", gpus[0].BusId)
+	assert.Equal(t, 32768, gpus[0].MemorySizeMB)
+	assert.Equal(t, "Default", gpus[0].ComputeMode)
+	assert.Equal(t, 1, gpus[1].Index)
+}
+
+func TestIluvatarPCIAddrCandidates(t *testing.T) {
+	cands := iluvatarPCIAddrCandidates("00000000:26:00.0")
+	assert.Equal(t, []string{"00000000:26:00.0", "0000:26:00.0", "26:00.0"}, cands)
+}

pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go

@@ -0,0 +1,245 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newKunlunxinXPUManager())
+}
+
+type kunlunxinXPUManager struct{}
+
+func newKunlunxinXPUManager() *kunlunxinXPUManager {
+	return &kunlunxinXPUManager{}
+}
+
+func (m *kunlunxinXPUManager) GetRegisterType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeKunlunxinXpu
+}
+
+func (m *kunlunxinXPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return getKunlunxinXPUs(m)
+}
+
+func (m *kunlunxinXPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	if dev.IsolatedDevice == nil {
+		return nil, nil, errors.Errorf("isolated device is nil")
+	}
+	iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+	if iDev == nil {
+		return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id)
+	}
+	gpuDev, ok := iDev.(*kunlunxinXPU)
+	if !ok {
+		return nil, nil, errors.Errorf("device %s is not kunlunxin xpu", dev.IsolatedDevice.Id)
+	}
+	ctrDevs := []*runtimeapi.Device{}
+	node := kunlunxinXpuDevNode(gpuDev.GetIndex())
+	if hygonPathExists(node) {
+		ctrDevs = append(ctrDevs, kunlunxinXpuDeviceSpec(node))
+	} else {
+		log.Warningf("kunlunxin xpu container device %s not found, skip", node)
+	}
+	return ctrDevs, kunlunxinXpuCommonDevices(), nil
+}
+
+func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	indices := collectKunlunxinXpuVisibleIndices(devs)
+	return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists)
+}
+
+type kunlunxinXPU struct {
+	manager isolated_device.IContainerDeviceManager
+	*BaseDevice
+
+	memSize  int
+	gpuIndex int
+	uuid     string
+}
+
+func (dev *kunlunxinXPU) GetMemorySize() int {
+	return dev.memSize
+}
+
+func (dev *kunlunxinXPU) GetIndex() int {
+	return dev.gpuIndex
+}
+
+func (dev *kunlunxinXPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
+	return dev.manager
+}
+
+func kunlunxinXpuCommonDevices() []*runtimeapi.Device {
+	devs := []*runtimeapi.Device{}
+	for _, p := range collectKunlunxinXpuCommonDevicePaths(hygonPathExists) {
+		devs = append(devs, kunlunxinXpuDeviceSpec(p))
+	}
+	return devs
+}
+
+func kunlunxinXreHome() string {
+	home := options.HostOptions.KunlunxinXreHome
+	if home == "" {
+		return defaultKunlunxinXreHome
+	}
+	return home
+}
+
+func kunlunxinXpuSmiPath() string {
+	p := options.HostOptions.KunlunxinXpuSmiPath
+	if p != "" {
+		return p
+	}
+	return defaultKunlunxinXpuSmiPath
+}
+
+func kunlunxinXpuLDLibraryPath() string {
+	libDir := kunlunxinXpuLibDir(kunlunxinXreHome(), hygonPathExists)
+	existing := os.Getenv("LD_LIBRARY_PATH")
+	if existing == "" {
+		return libDir
+	}
+	return existing + ":" + libDir
+}
+
+func runXpuSmi(args ...string) (string, error) {
+	smiPath := kunlunxinXpuSmiPath()
+	cmd := procutils.NewRemoteCommandAsFarAsPossible(smiPath, args...)
+	cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + kunlunxinXpuLDLibraryPath()})
+	out, err := cmd.Output()
+	if err != nil {
+		return string(out), errors.Wrapf(err, "xpu-smi %s", strings.Join(args, " "))
+	}
+	return string(out), nil
+}
+
+func lookupKunlunxinPCIDevice(busId, modelName string) *isolated_device.PCIDevice {
+	cands := kunlunxinXpuPCIAddrCandidates(busId)
+	for _, addr := range cands {
+		pciOutput, err := isolated_device.GetPCIStrByAddr(addr)
+		if err != nil || len(pciOutput) == 0 {
+			log.Warningf("kunlunxin xpu GetPCIStrByAddr %s: %v", addr, err)
+			continue
+		}
+		dev := isolated_device.NewPCIDevice2(pciOutput[0])
+		if modelName != "" {
+			dev.ModelName = modelName
+		}
+		return dev
+	}
+	fallbackAddr := busId
+	if len(cands) > 0 {
+		fallbackAddr = cands[len(cands)-1]
+	}
+	return &isolated_device.PCIDevice{
+		VendorId:   computeapi.KUNLUNXIN_VENDOR_ID,
+		VendorName: "KUNLUNXIN",
+		Addr:       fallbackAddr,
+		ModelName:  modelName,
+		DeviceName: modelName,
+	}
+}
+
+func collectKunlunxinXpuVisibleIndices(devs []*hostapi.ContainerDevice) []string {
+	indices := []string{}
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		devMan := iDev.GetContainerDeviceManager()
+		if _, ok := devMan.(*kunlunxinXPUManager); !ok {
+			continue
+		}
+		if dev.IsolatedDevice.Path != "" {
+			indices = append(indices, dev.IsolatedDevice.Path)
+			continue
+		}
+		if dev.IsolatedDevice.Index >= 0 {
+			indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index))
+		}
+	}
+	return indices
+}
+
+func getKunlunxinXPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) {
+	smiPath := kunlunxinXpuSmiPath()
+	if !hygonPathExists(smiPath) {
+		log.Infof("kunlunxin xpu probe skipped: xpu-smi not found at %s", smiPath)
+		return nil, nil
+	}
+
+	out, err := runXpuSmi()
+	if err != nil {
+		log.Warningf("xpu-smi failed: %v", err)
+		return nil, nil
+	}
+	parsed := parseXpuSmiTable(out)
+	if len(parsed) == 0 {
+		log.Infof("kunlunxin xpu probe: no devices parsed from xpu-smi")
+		return nil, nil
+	}
+
+	devs := make([]isolated_device.IDevice, 0, len(parsed))
+	for _, gpu := range parsed {
+		pciDev := lookupKunlunxinPCIDevice(gpu.BusId, gpu.Name)
+		indexStr := strconv.Itoa(gpu.Index)
+		node := kunlunxinXpuDevNode(gpu.Index)
+		if !hygonPathExists(node) {
+			log.Warningf("kunlunxin xpu %d: %s not found, still register by index", gpu.Index, node)
+		}
+		dev := &kunlunxinXPU{
+			manager:    manager,
+			BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1),
+			memSize:    gpu.MemorySizeMB,
+			gpuIndex:   gpu.Index,
+			uuid:       gpu.UUID,
+		}
+		if gpu.Name != "" {
+			dev.SetModelName(gpu.Name)
+		}
+		log.Infof("kunlunxin xpu idx=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s",
+			gpu.Index, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, node)
+		devs = append(devs, dev)
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	return devs, nil
+}

pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go

@@ -0,0 +1,208 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const (
+	defaultKunlunxinXreHome    = "/usr/local/xpu"
+	defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi"
+	kunlunxinXpuDevicePrefix   = "/dev/xpu"
+)
+
+var kunlunxinXpuCommonDevicePaths = []string{
+	"/dev/xpuctrl",
+}
+
+type parsedKunlunxinXPU struct {
+	Index        int
+	Name         string
+	BusId        string
+	MemorySizeMB int
+	UUID         string
+}
+
+var (
+	// |   0  P800 OAM           N/A   | 00000000:03:00.0 N/A |                    0 |
+	kunlunxinXpuBusLineRe = regexp.MustCompile(`(?i)^\|\s*(\d+)\s+(.+?)\s+N/A\s+\|\s*([0-9a-f]+:[0-9a-f]+:[0-9a-f]+\.[0-9a-f]+)`)
+	// | N/A   35C  N/A     87W / 400W |      0MiB / 98304MiB |      0%      Default |
+	kunlunxinXpuMemUsedTotalRe = regexp.MustCompile(`(?i)(\d+)\s*MiB\s*/\s*(\d+)\s*MiB`)
+	kunlunxinXpuNodeRe         = regexp.MustCompile(`^xpu(\d+)$`)
+)
+
+func kunlunxinXpuDevNode(index int) string {
+	return fmt.Sprintf("%s%d", kunlunxinXpuDevicePrefix, index)
+}
+
+func kunlunxinXpuDeviceSpec(devPath string) *runtimeapi.Device {
+	return &runtimeapi.Device{
+		ContainerPath: devPath,
+		HostPath:      devPath,
+		Permissions:   "rwm",
+	}
+}
+
+func collectKunlunxinXpuCommonDevicePaths(pathExists func(string) bool) []string {
+	if pathExists == nil {
+		return nil
+	}
+	out := make([]string, 0, len(kunlunxinXpuCommonDevicePaths))
+	for _, p := range kunlunxinXpuCommonDevicePaths {
+		if pathExists(p) {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func normalizeKunlunxinXreHome(xreHome string) string {
+	if xreHome == "" {
+		return defaultKunlunxinXreHome
+	}
+	return xreHome
+}
+
+func kunlunxinXpuLibDir(xreHome string, pathExists func(string) bool) string {
+	xreHome = normalizeKunlunxinXreHome(xreHome)
+	candidates := []string{
+		path.Join(xreHome, "so"),
+		path.Join(xreHome, "lib64"),
+		path.Join(xreHome, "lib"),
+	}
+	if pathExists == nil {
+		return candidates[0]
+	}
+	for _, p := range candidates {
+		if pathExists(p) {
+			return p
+		}
+	}
+	return candidates[0]
+}
+
+func parseXpuSmiTable(output string) []*parsedKunlunxinXPU {
+	var out []*parsedKunlunxinXPU
+	lines := strings.Split(output, "\n")
+	for i := 0; i < len(lines); i++ {
+		line := strings.TrimSpace(lines[i])
+		if line == "" {
+			continue
+		}
+		m := kunlunxinXpuBusLineRe.FindStringSubmatch(line)
+		if len(m) != 4 {
+			continue
+		}
+		idx, err := strconv.Atoi(m[1])
+		if err != nil {
+			continue
+		}
+		name := strings.TrimSpace(m[2])
+		busId := strings.TrimSpace(m[3])
+		mem := 0
+		// memory total is on the next data row
+		for j := i + 1; j < len(lines) && j <= i+3; j++ {
+			memLine := strings.TrimSpace(lines[j])
+			mm := kunlunxinXpuMemUsedTotalRe.FindStringSubmatch(memLine)
+			if len(mm) == 3 {
+				mem, _ = strconv.Atoi(mm[2])
+				break
+			}
+		}
+		out = append(out, &parsedKunlunxinXPU{
+			Index:        idx,
+			Name:         name,
+			BusId:        busId,
+			MemorySizeMB: mem,
+			UUID:         busId,
+		})
+	}
+	return out
+}
+
+func kunlunxinXpuPCIAddrCandidates(busId string) []string {
+	busId = strings.TrimSpace(busId)
+	if busId == "" {
+		return nil
+	}
+	cands := make([]string, 0, 3)
+	seen := map[string]bool{}
+	add := func(s string) {
+		if s == "" || seen[s] {
+			return
+		}
+		seen[s] = true
+		cands = append(cands, s)
+	}
+	add(busId)
+	parts := strings.Split(busId, ":")
+	if len(parts) == 3 {
+		domain, bus, fn := parts[0], parts[1], parts[2]
+		if len(domain) > 4 {
+			add(domain[len(domain)-4:] + ":" + bus + ":" + fn)
+		}
+		add(bus + ":" + fn)
+	}
+	return cands
+}
+
+func parseKunlunxinXpuNodeIndex(name string) (int, bool) {
+	m := kunlunxinXpuNodeRe.FindStringSubmatch(name)
+	if len(m) != 2 {
+		return 0, false
+	}
+	idx, err := strconv.Atoi(m[1])
+	if err != nil {
+		return 0, false
+	}
+	return idx, true
+}
+
+func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists func(string) bool) []*runtimeapi.KeyValue {
+	xreHome = normalizeKunlunxinXreHome(xreHome)
+	visible := strings.Join(indices, ",")
+	return []*runtimeapi.KeyValue{
+		{Key: "XPU_VISIBLE_DEVICES", Value: visible},
+		{Key: "LD_LIBRARY_PATH", Value: kunlunxinXpuLibDir(xreHome, pathExists)},
+	}
+}
+
+func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount {
+	xreHome = normalizeKunlunxinXreHome(xreHome)
+	if pathExists == nil || !pathExists(xreHome) {
+		return nil
+	}
+	return []*runtimeapi.Mount{
+		{
+			ContainerPath: xreHome,
+			HostPath:      xreHome,
+			Readonly:      true,
+		},
+	}
+}
+
+func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	if len(indices) == 0 {
+		return nil, nil
+	}
+	return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists)
+}

pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go

@@ -0,0 +1,111 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"yunion.io/x/onecloud/pkg/hostman/options"
+)
+
+func TestKunlunxinXpuDevNodeUsesIndex(t *testing.T) {
+	assert.Equal(t, "/dev/xpu0", kunlunxinXpuDevNode(0))
+	assert.Equal(t, "/dev/xpu7", kunlunxinXpuDevNode(7))
+}
+
+func TestCollectKunlunxinXpuCommonDevicePaths(t *testing.T) {
+	assert.Nil(t, collectKunlunxinXpuCommonDevicePaths(nil))
+	assert.Empty(t, collectKunlunxinXpuCommonDevicePaths(func(string) bool { return false }))
+	assert.Equal(t, []string{"/dev/xpuctrl"}, collectKunlunxinXpuCommonDevicePaths(func(string) bool { return true }))
+}
+
+func TestParseXpuSmiTable(t *testing.T) {
+	input := `Thu Sep 10 08:18:37 2026       
++-----------------------------------------------------------------------------+
+| XPU-SMI               Driver Version: 5.19.0.0     XPU-RT Version: 5.19.0   |
+|-------------------------------+----------------------+----------------------+
+| XPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | XPU-Util  Compute M. |
+|                               |             L3-Usage |            SR-IOV M. |
+|===============================+======================+======================|
+|   0  P800 OAM           N/A   | 00000000:03:00.0 N/A |                    0 |
+| N/A   35C  N/A     87W / 400W |      0MiB / 98304MiB |      0%      Default |
+|                               |      0MiB /    96MiB |             Disabled |
++-------------------------------+----------------------+----------------------+
+|   1  P800 OAM           N/A   | 00000000:05:00.0 N/A |                    0 |
+| N/A   37C  N/A     84W / 400W |      0MiB / 98304MiB |      0%      Default |
+|                               |      0MiB /    96MiB |             Disabled |
++-------------------------------+----------------------+----------------------+
+|   7  P800 OAM           N/A   | 00000000:A5:00.0 N/A |                    0 |
+| N/A   35C  N/A     85W / 400W |      0MiB / 98304MiB |      0%      Default |
+|                               |      0MiB /    96MiB |             Disabled |
++-------------------------------+----------------------+----------------------+
+`
+	gpus := parseXpuSmiTable(input)
+	require.Len(t, gpus, 3)
+	assert.Equal(t, 0, gpus[0].Index)
+	assert.Equal(t, "P800 OAM", gpus[0].Name)
+	assert.Equal(t, "00000000:03:00.0", gpus[0].BusId)
+	assert.Equal(t, "00000000:03:00.0", gpus[0].UUID)
+	assert.Equal(t, 98304, gpus[0].MemorySizeMB)
+	assert.Equal(t, 1, gpus[1].Index)
+	assert.Equal(t, "00000000:05:00.0", gpus[1].BusId)
+	assert.Equal(t, 7, gpus[2].Index)
+	assert.Equal(t, "00000000:A5:00.0", gpus[2].BusId)
+}
+
+func TestKunlunxinXpuPCIAddrCandidates(t *testing.T) {
+	cands := kunlunxinXpuPCIAddrCandidates("00000000:a8:00.0")
+	assert.Equal(t, []string{"00000000:a8:00.0", "0000:a8:00.0", "a8:00.0"}, cands)
+}
+
+func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
+	exists := func(p string) bool {
+		return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib"
+	}
+	envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists)
+	require.Len(t, envs, 2)
+	assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key)
+	assert.Equal(t, "0,3", envs[0].Value)
+	assert.Equal(t, "LD_LIBRARY_PATH", envs[1].Key)
+	assert.Equal(t, "/usr/local/xpu/lib", envs[1].Value)
+	require.Len(t, mounts, 1)
+	assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
+	assert.True(t, mounts[0].Readonly)
+
+	envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists)
+	assert.Nil(t, envs)
+	assert.Nil(t, mounts)
+}
+
+func TestParseKunlunxinXpuNodeIndex(t *testing.T) {
+	idx, ok := parseKunlunxinXpuNodeIndex("xpu3")
+	assert.True(t, ok)
+	assert.Equal(t, 3, idx)
+	_, ok = parseKunlunxinXpuNodeIndex("xpuctrl")
+	assert.False(t, ok)
+}
+
+func TestKunlunxinXpuSmiPathDefault(t *testing.T) {
+	orig := options.HostOptions.KunlunxinXpuSmiPath
+	defer func() { options.HostOptions.KunlunxinXpuSmiPath = orig }()
+	options.HostOptions.KunlunxinXpuSmiPath = ""
+	assert.Equal(t, "/usr/local/bin/xpu-smi", kunlunxinXpuSmiPath())
+	options.HostOptions.KunlunxinXpuSmiPath = "/opt/bin/xpu-smi"
+	assert.Equal(t, "/opt/bin/xpu-smi", kunlunxinXpuSmiPath())
+}

pkg/hostman/isolated_device/container_device/thead_ppu.go

@@ -0,0 +1,259 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func init() {
+	isolated_device.RegisterContainerDeviceManager(newTHeadPPUManager())
+}
+
+type tHeadPPUManager struct{}
+
+func newTHeadPPUManager() *tHeadPPUManager {
+	return &tHeadPPUManager{}
+}
+
+func (m *tHeadPPUManager) GetRegisterType() isolated_device.ContainerDeviceType {
+	return isolated_device.ContainerDeviceTypeTHeadPpu
+}
+
+func (m *tHeadPPUManager) ProbeDevices() ([]isolated_device.IDevice, error) {
+	return getTHeadPPUs(m)
+}
+
+func (m *tHeadPPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) {
+	return nil, nil
+}
+
+func (m *tHeadPPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) {
+	if dev.IsolatedDevice == nil {
+		return nil, nil, errors.Errorf("isolated device is nil")
+	}
+	iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+	if iDev == nil {
+		return nil, nil, errors.Errorf("device %s not found", dev.IsolatedDevice.Id)
+	}
+	gpuDev, ok := iDev.(*tHeadPPU)
+	if !ok {
+		return nil, nil, errors.Errorf("device %s is not t-head ppu", dev.IsolatedDevice.Id)
+	}
+	ctrDevs := []*runtimeapi.Device{}
+	node := tHeadPpuDevNode(gpuDev.GetIndex())
+	if hygonPathExists(node) {
+		ctrDevs = append(ctrDevs, tHeadPpuDeviceSpec(node))
+	} else {
+		log.Warningf("t-head ppu container device %s not found, skip", node)
+	}
+	return ctrDevs, tHeadPpuCommonDevices(), nil
+}
+
+func (m *tHeadPPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	indices := collectTHeadPpuVisibleIndices(devs)
+	return buildTHeadPpuExtraConfigures(indices, tHeadPpuSdkHome(), hygonPathExists)
+}
+
+type tHeadPPU struct {
+	manager isolated_device.IContainerDeviceManager
+	*BaseDevice
+
+	memSize  int
+	gpuIndex int
+	uuid     string
+}
+
+func (dev *tHeadPPU) GetMemorySize() int {
+	return dev.memSize
+}
+
+func (dev *tHeadPPU) GetIndex() int {
+	return dev.gpuIndex
+}
+
+func (dev *tHeadPPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager {
+	return dev.manager
+}
+
+func tHeadPpuCommonDevices() []*runtimeapi.Device {
+	devs := []*runtimeapi.Device{}
+	for _, p := range collectTHeadPpuCommonDevicePaths(hygonPathExists) {
+		devs = append(devs, tHeadPpuDeviceSpec(p))
+	}
+	return devs
+}
+
+func tHeadPpuSdkHome() string {
+	home := options.HostOptions.THeadPpuSdkHome
+	if home == "" {
+		return defaultTHeadPpuSdkHome
+	}
+	return home
+}
+
+func tHeadPpuSmiPath() string {
+	p := options.HostOptions.THeadPpuSmiPath
+	if p != "" {
+		return p
+	}
+	return defaultTHeadPpuSmiPath
+}
+
+func tHeadPpuLDLibraryPath() string {
+	libDir := tHeadPpuLibDir(tHeadPpuSdkHome(), hygonPathExists)
+	existing := os.Getenv("LD_LIBRARY_PATH")
+	if existing == "" {
+		return libDir
+	}
+	return existing + ":" + libDir
+}
+
+func runPpuSmi(args ...string) (string, error) {
+	smiPath := tHeadPpuSmiPath()
+	cmd := procutils.NewRemoteCommandAsFarAsPossible(smiPath, args...)
+	cmd.SetEnv([]string{"LD_LIBRARY_PATH=" + tHeadPpuLDLibraryPath()})
+	out, err := cmd.Output()
+	if err != nil {
+		return string(out), errors.Wrapf(err, "ppu-smi %s", strings.Join(args, " "))
+	}
+	return string(out), nil
+}
+
+func lookupTHeadPCIDevice(busId, modelName string) *isolated_device.PCIDevice {
+	cands := tHeadPpuPCIAddrCandidates(busId)
+	for _, addr := range cands {
+		pciOutput, err := isolated_device.GetPCIStrByAddr(addr)
+		if err != nil || len(pciOutput) == 0 {
+			log.Warningf("t-head ppu GetPCIStrByAddr %s: %v", addr, err)
+			continue
+		}
+		dev := isolated_device.NewPCIDevice2(pciOutput[0])
+		if modelName != "" {
+			dev.ModelName = modelName
+		}
+		return dev
+	}
+	fallbackAddr := busId
+	if len(cands) > 0 {
+		fallbackAddr = cands[len(cands)-1]
+	}
+	return &isolated_device.PCIDevice{
+		VendorId:   computeapi.THEAD_VENDOR_ID,
+		VendorName: "THEAD",
+		Addr:       fallbackAddr,
+		ModelName:  modelName,
+		DeviceName: modelName,
+	}
+}
+
+func collectTHeadPpuVisibleIndices(devs []*hostapi.ContainerDevice) []string {
+	indices := []string{}
+	for _, dev := range devs {
+		if dev.IsolatedDevice == nil {
+			continue
+		}
+		iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id)
+		if iDev == nil {
+			continue
+		}
+		devMan := iDev.GetContainerDeviceManager()
+		if _, ok := devMan.(*tHeadPPUManager); !ok {
+			continue
+		}
+		if dev.IsolatedDevice.Path != "" {
+			indices = append(indices, dev.IsolatedDevice.Path)
+			continue
+		}
+		if dev.IsolatedDevice.Index >= 0 {
+			indices = append(indices, strconv.Itoa(dev.IsolatedDevice.Index))
+		}
+	}
+	return indices
+}
+
+func getTHeadPPUs(manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) {
+	smiPath := tHeadPpuSmiPath()
+	if !hygonPathExists(smiPath) {
+		log.Infof("t-head ppu probe skipped: ppu-smi not found at %s", smiPath)
+		return nil, nil
+	}
+
+	parsed := []*parsedTHeadPPU{}
+	queryOut, err := runPpuSmi("--query-ppu=index,name,uuid,pci.bus_id,memory.total", "--format=csv,noheader,nounits")
+	if err != nil {
+		log.Warningf("ppu-smi --query-ppu failed: %v", err)
+	} else {
+		parsed = parsePpuSmiQueryCSV(queryOut)
+	}
+
+	listOut, err := runPpuSmi("-L")
+	if err != nil {
+		log.Warningf("ppu-smi -L failed: %v", err)
+	} else {
+		parsed = mergeTHeadPpuProbe(parsed, parsePpuSmiList(listOut))
+		if len(parsed) == 0 {
+			for idx, uuid := range parsePpuSmiList(listOut) {
+				parsed = append(parsed, &parsedTHeadPPU{Index: idx, UUID: uuid})
+			}
+		}
+	}
+
+	if len(parsed) == 0 {
+		log.Infof("t-head ppu probe: no devices parsed from ppu-smi")
+		return nil, nil
+	}
+
+	devs := make([]isolated_device.IDevice, 0, len(parsed))
+	for _, gpu := range parsed {
+		pciDev := lookupTHeadPCIDevice(gpu.BusId, gpu.Name)
+		indexStr := strconv.Itoa(gpu.Index)
+		node := tHeadPpuDevNode(gpu.Index)
+		if !hygonPathExists(node) {
+			log.Warningf("t-head ppu %d: %s not found, still register by index", gpu.Index, node)
+		}
+		dev := &tHeadPPU{
+			manager:    manager,
+			BaseDevice: NewBaseDevice(pciDev, computeapi.GPU_TYPE, indexStr, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, 1),
+			memSize:    gpu.MemorySizeMB,
+			gpuIndex:   gpu.Index,
+			uuid:       gpu.UUID,
+		}
+		if gpu.Name != "" {
+			dev.SetModelName(gpu.Name)
+		}
+		log.Infof("t-head ppu idx=%d model=%s pci=%s uuid=%s memMiB=%d path=%s node=%s",
+			gpu.Index, gpu.Name, pciDev.Addr, gpu.UUID, gpu.MemorySizeMB, indexStr, node)
+		devs = append(devs, dev)
+	}
+	if len(devs) == 0 {
+		return nil, nil
+	}
+	return devs, nil
+}

pkg/hostman/isolated_device/container_device/thead_ppusmi.go

@@ -0,0 +1,242 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"fmt"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+const (
+	defaultTHeadPpuSdkHome = "/usr/local/PPU_SDK"
+	defaultTHeadPpuSmiPath = "/usr/local/bin/ppu-smi"
+	tHeadPpuDevicePrefix   = "/dev/alixpu_ppu"
+)
+
+var tHeadPpuCommonDevicePaths = []string{
+	"/dev/alixpu",
+	"/dev/alixpu_ctl",
+	"/dev/alixpu_sep",
+}
+
+type parsedTHeadPPU struct {
+	Index        int
+	Name         string
+	BusId        string
+	MemorySizeMB int
+	UUID         string
+}
+
+var (
+	tHeadPpuListRe  = regexp.MustCompile(`(?i)^PPU\s+(\d+):\s+.+\(UUID:\s*([^)]+)\)`)
+	tHeadPpuMemRe   = regexp.MustCompile(`(\d+)`)
+	tHeadPpuNodeRe  = regexp.MustCompile(`^alixpu_ppu(\d+)$`)
+	tHeadPpuQueryRe = regexp.MustCompile(`index,name,uuid,pci\.bus_id,memory\.total`)
+)
+
+func tHeadPpuDevNode(index int) string {
+	return fmt.Sprintf("%s%d", tHeadPpuDevicePrefix, index)
+}
+
+func tHeadPpuDeviceSpec(devPath string) *runtimeapi.Device {
+	return &runtimeapi.Device{
+		ContainerPath: devPath,
+		HostPath:      devPath,
+		Permissions:   "rwm",
+	}
+}
+
+func collectTHeadPpuCommonDevicePaths(pathExists func(string) bool) []string {
+	if pathExists == nil {
+		return nil
+	}
+	out := make([]string, 0, len(tHeadPpuCommonDevicePaths))
+	for _, p := range tHeadPpuCommonDevicePaths {
+		if pathExists(p) {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func normalizeTHeadPpuSdkHome(sdkHome string) string {
+	if sdkHome == "" {
+		return defaultTHeadPpuSdkHome
+	}
+	return sdkHome
+}
+
+func tHeadPpuLibDir(sdkHome string, pathExists func(string) bool) string {
+	sdkHome = normalizeTHeadPpuSdkHome(sdkHome)
+	lib64 := path.Join(sdkHome, "lib64")
+	if pathExists == nil || pathExists(lib64) {
+		return lib64
+	}
+	return path.Join(sdkHome, "lib")
+}
+
+func parsePpuSmiQueryCSV(output string) []*parsedTHeadPPU {
+	var out []*parsedTHeadPPU
+	for _, raw := range strings.Split(output, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" {
+			continue
+		}
+		if tHeadPpuQueryRe.MatchString(strings.ToLower(strings.ReplaceAll(line, " ", ""))) {
+			continue
+		}
+		if strings.HasPrefix(strings.ToLower(line), "timestamp") {
+			continue
+		}
+		cols := splitCSVLine(line)
+		if len(cols) < 5 {
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(cols[0]))
+		if err != nil {
+			continue
+		}
+		mem := 0
+		if m := tHeadPpuMemRe.FindStringSubmatch(cols[4]); len(m) == 2 {
+			mem, _ = strconv.Atoi(m[1])
+		}
+		out = append(out, &parsedTHeadPPU{
+			Index:        idx,
+			Name:         strings.TrimSpace(cols[1]),
+			UUID:         strings.TrimSpace(cols[2]),
+			BusId:        strings.TrimSpace(cols[3]),
+			MemorySizeMB: mem,
+		})
+	}
+	return out
+}
+
+func splitCSVLine(line string) []string {
+	parts := strings.Split(line, ",")
+	cols := make([]string, 0, len(parts))
+	for _, p := range parts {
+		cols = append(cols, strings.TrimSpace(p))
+	}
+	return cols
+}
+
+func parsePpuSmiList(output string) map[int]string {
+	ret := map[int]string{}
+	for _, raw := range strings.Split(output, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" {
+			continue
+		}
+		m := tHeadPpuListRe.FindStringSubmatch(line)
+		if len(m) != 3 {
+			continue
+		}
+		idx, err := strconv.Atoi(m[1])
+		if err != nil {
+			continue
+		}
+		ret[idx] = strings.TrimSpace(m[2])
+	}
+	return ret
+}
+
+func mergeTHeadPpuProbe(table []*parsedTHeadPPU, uuids map[int]string) []*parsedTHeadPPU {
+	for _, gpu := range table {
+		if gpu == nil {
+			continue
+		}
+		if gpu.UUID != "" {
+			continue
+		}
+		if uuid, ok := uuids[gpu.Index]; ok {
+			gpu.UUID = uuid
+		}
+	}
+	return table
+}
+
+func tHeadPpuPCIAddrCandidates(busId string) []string {
+	busId = strings.TrimSpace(busId)
+	if busId == "" {
+		return nil
+	}
+	cands := make([]string, 0, 3)
+	seen := map[string]bool{}
+	add := func(s string) {
+		if s == "" || seen[s] {
+			return
+		}
+		seen[s] = true
+		cands = append(cands, s)
+	}
+	add(busId)
+	parts := strings.Split(busId, ":")
+	if len(parts) == 3 {
+		domain, bus, fn := parts[0], parts[1], parts[2]
+		if len(domain) > 4 {
+			add(domain[len(domain)-4:] + ":" + bus + ":" + fn)
+		}
+		add(bus + ":" + fn)
+	}
+	return cands
+}
+
+func parseTHeadPpuNodeIndex(name string) (int, bool) {
+	m := tHeadPpuNodeRe.FindStringSubmatch(name)
+	if len(m) != 2 {
+		return 0, false
+	}
+	idx, err := strconv.Atoi(m[1])
+	if err != nil {
+		return 0, false
+	}
+	return idx, true
+}
+
+func buildTHeadPpuRuntimeEnvs(indices []string, sdkHome string, pathExists func(string) bool) []*runtimeapi.KeyValue {
+	sdkHome = normalizeTHeadPpuSdkHome(sdkHome)
+	visible := strings.Join(indices, ",")
+	return []*runtimeapi.KeyValue{
+		{Key: "CUDA_VISIBLE_DEVICES", Value: visible},
+		{Key: "NVIDIA_VISIBLE_DEVICES", Value: visible},
+		{Key: "LD_LIBRARY_PATH", Value: tHeadPpuLibDir(sdkHome, pathExists)},
+	}
+}
+
+func buildTHeadPpuRuntimeMounts(sdkHome string, pathExists func(string) bool) []*runtimeapi.Mount {
+	sdkHome = normalizeTHeadPpuSdkHome(sdkHome)
+	if pathExists == nil || !pathExists(sdkHome) {
+		return nil
+	}
+	return []*runtimeapi.Mount{
+		{
+			ContainerPath: sdkHome,
+			HostPath:      sdkHome,
+			Readonly:      true,
+		},
+	}
+}
+
+func buildTHeadPpuExtraConfigures(indices []string, sdkHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
+	if len(indices) == 0 {
+		return nil, nil
+	}
+	return buildTHeadPpuRuntimeEnvs(indices, sdkHome, pathExists), buildTHeadPpuRuntimeMounts(sdkHome, pathExists)
+}

pkg/hostman/isolated_device/container_device/thead_ppusmi_test.go

@@ -0,0 +1,108 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container_device
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"yunion.io/x/onecloud/pkg/hostman/options"
+)
+
+func TestTHeadPpuDevNodeUsesIndexNotMinor(t *testing.T) {
+	assert.Equal(t, "/dev/alixpu_ppu0", tHeadPpuDevNode(0))
+	assert.Equal(t, "/dev/alixpu_ppu3", tHeadPpuDevNode(3))
+	assert.NotEqual(t, "/dev/alixpu_ppu4", tHeadPpuDevNode(3))
+}
+
+func TestCollectTHeadPpuCommonDevicePaths(t *testing.T) {
+	assert.Nil(t, collectTHeadPpuCommonDevicePaths(nil))
+	assert.Empty(t, collectTHeadPpuCommonDevicePaths(func(string) bool { return false }))
+	assert.Equal(t, []string{
+		"/dev/alixpu",
+		"/dev/alixpu_ctl",
+		"/dev/alixpu_sep",
+	}, collectTHeadPpuCommonDevicePaths(func(string) bool { return true }))
+	assert.Equal(t, []string{"/dev/alixpu", "/dev/alixpu_sep"}, collectTHeadPpuCommonDevicePaths(func(p string) bool {
+		return p != "/dev/alixpu_ctl"
+	}))
+}
+
+func TestParsePpuSmiQueryCSV(t *testing.T) {
+	input := `index, name, uuid, pci.bus_id, memory.total [MiB]
+0, PPU-ZW810E, GPU-019e2226-4211-0208-0000-000000ab261d, 00000000:a8:00.0, 98304 MiB
+1, PPU-ZW810E, GPU-019e2226-84c1-0200-0000-0000c020e701, 00000000:a7:00.0, 98304`
+	gpus := parsePpuSmiQueryCSV(input)
+	require.Len(t, gpus, 2)
+	assert.Equal(t, 0, gpus[0].Index)
+	assert.Equal(t, "PPU-ZW810E", gpus[0].Name)
+	assert.Equal(t, "GPU-019e2226-4211-0208-0000-000000ab261d", gpus[0].UUID)
+	assert.Equal(t, "00000000:a8:00.0", gpus[0].BusId)
+	assert.Equal(t, 98304, gpus[0].MemorySizeMB)
+	assert.Equal(t, 1, gpus[1].Index)
+}
+
+func TestParsePpuSmiList(t *testing.T) {
+	input := `PPU 0: PPU-ZW810E (UUID: GPU-019ea108-c110-0828-0000-0000c07e1a46)
+PPU 1: PPU (UUID: GPU-019ea108-c120-040c-0000-0000c0267f1e)`
+	uuids := parsePpuSmiList(input)
+	assert.Equal(t, "GPU-019ea108-c110-0828-0000-0000c07e1a46", uuids[0])
+	assert.Equal(t, "GPU-019ea108-c120-040c-0000-0000c0267f1e", uuids[1])
+}
+
+func TestTHeadPpuPCIAddrCandidates(t *testing.T) {
+	cands := tHeadPpuPCIAddrCandidates("00000000:a8:00.0")
+	assert.Equal(t, []string{"00000000:a8:00.0", "0000:a8:00.0", "a8:00.0"}, cands)
+}
+
+func TestBuildTHeadPpuExtraConfigures(t *testing.T) {
+	exists := func(p string) bool {
+		return p == "/usr/local/PPU_SDK" || p == "/usr/local/PPU_SDK/lib"
+	}
+	envs, mounts := buildTHeadPpuExtraConfigures([]string{"0", "3"}, "/usr/local/PPU_SDK", exists)
+	require.Len(t, envs, 3)
+	assert.Equal(t, "CUDA_VISIBLE_DEVICES", envs[0].Key)
+	assert.Equal(t, "0,3", envs[0].Value)
+	assert.Equal(t, "NVIDIA_VISIBLE_DEVICES", envs[1].Key)
+	assert.Equal(t, "0,3", envs[1].Value)
+	assert.Equal(t, "LD_LIBRARY_PATH", envs[2].Key)
+	assert.Equal(t, "/usr/local/PPU_SDK/lib", envs[2].Value)
+	require.Len(t, mounts, 1)
+	assert.Equal(t, "/usr/local/PPU_SDK", mounts[0].HostPath)
+	assert.True(t, mounts[0].Readonly)
+
+	envs, mounts = buildTHeadPpuExtraConfigures(nil, "/usr/local/PPU_SDK", exists)
+	assert.Nil(t, envs)
+	assert.Nil(t, mounts)
+}
+
+func TestParseTHeadPpuNodeIndex(t *testing.T) {
+	idx, ok := parseTHeadPpuNodeIndex("alixpu_ppu3")
+	assert.True(t, ok)
+	assert.Equal(t, 3, idx)
+	_, ok = parseTHeadPpuNodeIndex("alixpu")
+	assert.False(t, ok)
+}
+
+func TestTHeadPpuSmiPathDefault(t *testing.T) {
+	orig := options.HostOptions.THeadPpuSmiPath
+	defer func() { options.HostOptions.THeadPpuSmiPath = orig }()
+	options.HostOptions.THeadPpuSmiPath = ""
+	assert.Equal(t, "/usr/local/bin/ppu-smi", tHeadPpuSmiPath())
+	options.HostOptions.THeadPpuSmiPath = "/opt/bin/ppu-smi"
+	assert.Equal(t, "/opt/bin/ppu-smi", tHeadPpuSmiPath())
+}

pkg/hostman/isolated_device/isolated_device.go

@@ -324,6 +324,81 @@ func (man *isolatedDeviceManager) probeContainerHygonDCUs(enable, enableHami boo
 	log.Infof("==== hygon dcu probe finished: total %d devices", len(devs))
 }
 
+func (man *isolatedDeviceManager) probeContainerIluvatarGPUs(enable bool) {
+	if !enable {
+		log.Infof("iluvatar gpu probe skipped: enable_container_iluvatar_gpu=false")
+		return
+	}
+	devman, err := GetContainerDeviceManager(ContainerDeviceTypeIluvatarGpu)
+	if err != nil {
+		log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeIluvatarGpu, err)
+		return
+	}
+	devs, err := devman.ProbeDevices()
+	if err != nil {
+		log.Warningf("Probe container iluvatar gpu devices: %v", err)
+		return
+	}
+	if len(devs) == 0 {
+		log.Infof("iluvatar gpu probe finished: no devices found")
+		return
+	}
+	for idx, dev := range devs {
+		man.devices = append(man.devices, dev)
+		log.Infof("Add Container iluvatar GPU device: %d => %#v", idx, dev)
+	}
+}
+
+func (man *isolatedDeviceManager) probeContainerTHeadPPUs(enable bool) {
+	if !enable {
+		log.Infof("t-head ppu probe skipped: enable_container_t_head_ppu=false")
+		return
+	}
+	devman, err := GetContainerDeviceManager(ContainerDeviceTypeTHeadPpu)
+	if err != nil {
+		log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeTHeadPpu, err)
+		return
+	}
+	devs, err := devman.ProbeDevices()
+	if err != nil {
+		log.Warningf("Probe container t-head ppu devices: %v", err)
+		return
+	}
+	if len(devs) == 0 {
+		log.Infof("t-head ppu probe finished: no devices found")
+		return
+	}
+	for idx, dev := range devs {
+		man.devices = append(man.devices, dev)
+		log.Infof("Add Container t-head PPU device: %d => %#v", idx, dev)
+	}
+}
+
+func (man *isolatedDeviceManager) probeContainerKunlunxinXPUs(enable bool) {
+	if !enable {
+		log.Infof("kunlunxin xpu probe skipped: enable_container_kunlunxin_xpu=false")
+		return
+	}
+	devman, err := GetContainerDeviceManager(ContainerDeviceTypeKunlunxinXpu)
+	if err != nil {
+		log.Errorf("no container device manager %s found: %v", ContainerDeviceTypeKunlunxinXpu, err)
+		return
+	}
+	devs, err := devman.ProbeDevices()
+	if err != nil {
+		log.Warningf("Probe container kunlunxin xpu devices: %v", err)
+		return
+	}
+	if len(devs) == 0 {
+		log.Infof("kunlunxin xpu probe finished: no devices found")
+		return
+	}
+	for idx, dev := range devs {
+		man.devices = append(man.devices, dev)
+		log.Infof("Add Container kunlunxin XPU device: %d => %#v", idx, dev)
+	}
+}
+
 func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgpuPFs []string, enableWhitelist bool, whitelistModels []IsolatedDeviceModel) {
 	if skipGPUs {
 		return
@@ -498,6 +573,9 @@ type SIsolatedDeviceProbeOptions struct {
 	EnableContainerAscendNpuHAMI bool
 	EnableContainerHygonDCU      bool
 	EnableContainerHygonDCUHAMI  bool
+	EnableContainerIluvatarGPU   bool
+	EnableContainerTHeadPPU      bool
+	EnableContainerKunlunxinXPU  bool
 	EnableWhitelist              bool
 
 	SriovNics, OvsOffloadNics []HostNic
@@ -515,6 +593,9 @@ func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOpti
 		man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps)
 		man.probeContainerAscendNPUs(opts.EnableContainerAscendNpu, opts.EnableContainerAscendNpuHAMI)
 		man.probeContainerHygonDCUs(opts.EnableContainerHygonDCU, opts.EnableContainerHygonDCUHAMI)
+		man.probeContainerIluvatarGPUs(opts.EnableContainerIluvatarGPU)
+		man.probeContainerTHeadPPUs(opts.EnableContainerTHeadPPU)
+		man.probeContainerKunlunxinXPUs(opts.EnableContainerKunlunxinXPU)
 	} else {
 		log.Infof("==== ProbePCIDevices: not container host, hygon container probe will NOT run (use host_type=container for hygon dcu)")
 		devModels, err := man.getCustomIsolatedDeviceModels()

pkg/hostman/options/options.go

@@ -288,6 +288,18 @@ type SHostOptions struct {
 	HygonVdevConfDir            string `help:"hygon vdcu config directory" default:"/etc/vdev"`
 	HygonVgpuCacheDir           string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"`
 
+	EnableContainerIluvatarGPU bool   `help:"enable container iluvatar gpu" default:"true"`
+	IluvatarCorexHome          string `help:"iluvatar corex home" default:"/usr/local/corex-4.4.0"`
+	IluvatarIxsmiPath          string `help:"iluvatar ixsmi path" default:"/usr/local/corex-4.4.0/bin/ixsmi"`
+
+	EnableContainerTHeadPPU bool   `help:"enable container t-head ppu" default:"true"`
+	THeadPpuSdkHome         string `help:"t-head ppu sdk home" default:"/usr/local/PPU_SDK"`
+	THeadPpuSmiPath         string `help:"t-head ppu-smi path" default:"/usr/local/bin/ppu-smi"`
+
+	EnableContainerKunlunxinXPU bool   `help:"enable container kunlunxin xpu" default:"true"`
+	KunlunxinXreHome            string `help:"kunlunxin xre home" default:"/usr/local/xpu"`
+	KunlunxinXpuSmiPath         string `help:"kunlunxin xpu-smi path" default:"/usr/local/bin/xpu-smi"`
+
 	EnableDirtyRecoverySeconds int  `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"`
 	EnableContainerCniPortmap  bool `help:"Use container cni portmap plugin" default:"false"`
 	DisableReconcileContainer  bool `help:"disable reconcile container" default:"false"`

pkg/hostman/system_service/telegraf.go

@@ -38,8 +38,11 @@ const (
 	TELEGRAF_INPUT_NETDEV              = "ni_rsrc_mon"
 	TELEGRAF_INPUT_VASMI               = "vasmi"
 	TELEGRAF_INPUT_HYSMI               = "hysmi"
+	TELEGRAF_INPUT_IXSMI               = "ixsmi"
+	TELEGRAF_INPUT_PPUSMI              = "ppusmi"
 	TELEGRAF_INPUT_NVIDIASMI           = "nvidia-smi"
 	TELEGRAF_INPUT_NPUSMI              = "npu-smi"
+	TELEGRAF_INPUT_CONF_LIB_PATH       = "lib_path"
 )
 
 type STelegraf struct {
@@ -346,6 +349,26 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string {
 		conf += "\n"
 	}
 
+	if ixsmi, ok := kwargs[TELEGRAF_INPUT_IXSMI]; ok {
+		ixsmiMap, _ := ixsmi.(map[string]interface{})
+		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_IXSMI)
+		conf += fmt.Sprintf("  bin_path = \"%s\"\n", ixsmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string))
+		if libPath, _ := ixsmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" {
+			conf += fmt.Sprintf("  lib_path = \"%s\"\n", libPath)
+		}
+		conf += "\n"
+	}
+
+	if ppusmi, ok := kwargs[TELEGRAF_INPUT_PPUSMI]; ok {
+		ppusmiMap, _ := ppusmi.(map[string]interface{})
+		conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_PPUSMI)
+		conf += fmt.Sprintf("  bin_path = \"%s\"\n", ppusmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string))
+		if libPath, _ := ppusmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" {
+			conf += fmt.Sprintf("  lib_path = \"%s\"\n", libPath)
+		}
+		conf += "\n"
+	}
+
 	if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok {
 		conf += "[[inputs.nvidia_smi]]\n"
 		conf += "\n"

pkg/llm/drivers/llm_container/vllm.go

@@ -430,6 +430,9 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo
 	modelPath := models.PickContainerModelMountPath(models.CollectContainerModelMountPaths(llm, sku), preferred)
 	hasMountedModels := modelPath != "" || len(postOverlays) > 0 || models.SkuHasLocalHostPathModel(sku)
 	hygon := models.HasHygonDevices(llm, sku)
+	iluvatar := models.HasIluvatarDevices(llm, sku)
+	thead := models.HasTHeadDevices(llm, sku)
+	kunlunxin := models.HasKunlunxinDevices(llm, sku)
 	startScript := buildVLLMEntrypointScript(modelPath, tensorParallelSize, backendParameters, effSpec, hygon)
 	envs := []*commonapi.ContainerKeyValue{
 		{
@@ -462,7 +465,7 @@ func (v *vllm) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *mo
 			Envs:              envs,
 		},
 	}
-	if hygon {
+	if hygon || iluvatar || thead || kunlunxin {
 		spec.Command = []string{"/bin/bash", "-c"}
 	}
 	if hasMountedModels {

pkg/llm/drivers/llm_container/vllm_hygon_test.go

@@ -82,3 +82,111 @@ func TestVLLMGetContainerSpecNvidiaNoHygonRuntime(t *testing.T) {
 		t.Fatalf("args should not source dtk env for nvidia, got %#v", spec.Args)
 	}
 }
+
+func TestVLLMGetContainerSpecIluvatarRuntime(t *testing.T) {
+	v := newVLLM().(*vllm)
+	hostPaths := api.HostPaths{
+		{
+			Type: "directory",
+			Path: "/data/model/Qwen3.5-122B-A10B-w4a8",
+			Containers: api.ContainerHostPathRelations{
+				"0": {MountPath: "/data/model/Qwen3.5-122B-A10B-w4a8"},
+			},
+		},
+	}
+	sku := &models.SLLMSku{
+		SLLMSkuBase: models.SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+			},
+		},
+		LLMType:   string(api.LLM_CONTAINER_VLLM),
+		Source:    api.LLM_MODEL_SOURCE_LOCAL_PATH,
+		LocalPath: "/data/model/Qwen3.5-122B-A10B-w4a8",
+	}
+	sku.HostPaths = &hostPaths
+	image := &models.SLLMImage{}
+	out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "")
+	if out == nil {
+		t.Fatal("expected container spec")
+	}
+	spec := &out.ContainerSpec
+	if spec.Capabilities != nil {
+		t.Fatalf("expected nil capabilities for iluvatar, got %#v", spec.Capabilities)
+	}
+	if spec.SecurityContext != nil {
+		t.Fatalf("expected nil security context for iluvatar, got %#v", spec.SecurityContext)
+	}
+	if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" {
+		t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command)
+	}
+	if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") {
+		t.Fatalf("args should not source dtk env for iluvatar, got %#v", spec.Args)
+	}
+	if len(spec.Args) == 0 || !strings.Contains(spec.Args[0], "--tensor-parallel-size 4") {
+		t.Fatalf("args = %#v, want tensor-parallel-size 4", spec.Args)
+	}
+}
+
+func TestVLLMGetContainerSpecTHeadRuntime(t *testing.T) {
+	v := newVLLM().(*vllm)
+	sku := &models.SLLMSku{
+		SLLMSkuBase: models.SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_THEAD_PPU},
+				{DevType: computeapi.CONTAINER_DEV_THEAD_PPU},
+			},
+		},
+	}
+	image := &models.SLLMImage{}
+	out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "")
+	if out == nil {
+		t.Fatal("expected container spec")
+	}
+	spec := &out.ContainerSpec
+	if spec.Capabilities != nil {
+		t.Fatalf("expected nil capabilities for t-head, got %#v", spec.Capabilities)
+	}
+	if spec.SecurityContext != nil {
+		t.Fatalf("expected nil security context for t-head, got %#v", spec.SecurityContext)
+	}
+	if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" {
+		t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command)
+	}
+	if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") {
+		t.Fatalf("args should not source dtk env for t-head, got %#v", spec.Args)
+	}
+}
+
+func TestVLLMGetContainerSpecKunlunxinRuntime(t *testing.T) {
+	v := newVLLM().(*vllm)
+	sku := &models.SLLMSku{
+		SLLMSkuBase: models.SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU},
+				{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU},
+			},
+		},
+	}
+	image := &models.SLLMImage{}
+	out := v.GetContainerSpec(context.Background(), nil, image, sku, nil, nil, "")
+	if out == nil {
+		t.Fatal("expected container spec")
+	}
+	spec := &out.ContainerSpec
+	if spec.Capabilities != nil {
+		t.Fatalf("expected nil capabilities for kunlunxin, got %#v", spec.Capabilities)
+	}
+	if spec.SecurityContext != nil {
+		t.Fatalf("expected nil security context for kunlunxin, got %#v", spec.SecurityContext)
+	}
+	if len(spec.Command) != 2 || spec.Command[0] != "/bin/bash" || spec.Command[1] != "-c" {
+		t.Fatalf("command = %#v, want [/bin/bash -c]", spec.Command)
+	}
+	if len(spec.Args) > 0 && strings.Contains(spec.Args[0], "/opt/dtk/env.sh") {
+		t.Fatalf("args should not source dtk env for kunlunxin, got %#v", spec.Args)
+	}
+}

pkg/llm/models/llm_base.go

@@ -403,6 +403,57 @@ func HasHygonDevices(llm *SLLM, sku *SLLMSku) bool {
 	return false
 }
 
+// HasIluvatarDevices reports whether effective devices include Iluvatar GPU.
+func HasIluvatarDevices(llm *SLLM, sku *SLLMSku) bool {
+	devs := GetEffectiveDevices(llm, sku)
+	if devs == nil {
+		return false
+	}
+	for _, d := range *devs {
+		if strings.EqualFold(d.Vendor, "ILUVATAR") {
+			return true
+		}
+		if d.DevType == computeapi.CONTAINER_DEV_ILUVATAR_GPU {
+			return true
+		}
+	}
+	return false
+}
+
+// HasTHeadDevices reports whether effective devices include T-Head PPU.
+func HasTHeadDevices(llm *SLLM, sku *SLLMSku) bool {
+	devs := GetEffectiveDevices(llm, sku)
+	if devs == nil {
+		return false
+	}
+	for _, d := range *devs {
+		if strings.EqualFold(d.Vendor, "THEAD") {
+			return true
+		}
+		if d.DevType == computeapi.CONTAINER_DEV_THEAD_PPU {
+			return true
+		}
+	}
+	return false
+}
+
+// HasKunlunxinDevices reports whether effective devices include Kunlunxin XPU.
+func HasKunlunxinDevices(llm *SLLM, sku *SLLMSku) bool {
+	devs := GetEffectiveDevices(llm, sku)
+	if devs == nil {
+		return false
+	}
+	for _, d := range *devs {
+		if strings.EqualFold(d.Vendor, "KUNLUNXIN") {
+			return true
+		}
+		if d.DevType == computeapi.CONTAINER_DEV_KUNLUNXIN_XPU {
+			return true
+		}
+	}
+	return false
+}
+
 // GetEffectiveHostPaths returns the host_paths to apply with llm's override taking priority over sku.
 func GetEffectiveHostPaths(llm *SLLM, sku *SLLMSku) *api.HostPaths {
 	var llmBase *SLLMBase

pkg/llm/models/llm_devices_test.go

@@ -63,3 +63,141 @@ func TestHasHygonDevices(t *testing.T) {
 		t.Fatal("expected normalized GPU+HYGON vendor sku to be detected as Hygon")
 	}
 }
+
+func TestHasIluvatarDevices(t *testing.T) {
+	iluvatarSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+			},
+		},
+	}
+	if !HasIluvatarDevices(nil, iluvatarSku) {
+		t.Fatal("expected Iluvatar GPU sku to be detected")
+	}
+
+	nvSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU},
+			},
+		},
+	}
+	if HasIluvatarDevices(nil, nvSku) {
+		t.Fatal("expected NVIDIA sku not to be detected as Iluvatar")
+	}
+
+	llm := &SLLM{
+		SLLMBase: SLLMBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+			},
+		},
+	}
+	if !HasIluvatarDevices(llm, nvSku) {
+		t.Fatal("expected llm device override to win over sku")
+	}
+
+	normalizedSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.GPU_TYPE, Vendor: "ILUVATAR", Model: "BI-V150S"},
+			},
+		},
+	}
+	if !HasIluvatarDevices(nil, normalizedSku) {
+		t.Fatal("expected normalized GPU+ILUVATAR vendor sku to be detected as Iluvatar")
+	}
+}
+
+func TestHasTHeadDevices(t *testing.T) {
+	theadSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_THEAD_PPU},
+			},
+		},
+	}
+	if !HasTHeadDevices(nil, theadSku) {
+		t.Fatal("expected T-Head PPU sku to be detected")
+	}
+
+	nvSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU},
+			},
+		},
+	}
+	if HasTHeadDevices(nil, nvSku) {
+		t.Fatal("expected NVIDIA sku not to be detected as T-Head")
+	}
+
+	llm := &SLLM{
+		SLLMBase: SLLMBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_THEAD_PPU},
+			},
+		},
+	}
+	if !HasTHeadDevices(llm, nvSku) {
+		t.Fatal("expected llm device override to win over sku")
+	}
+
+	normalizedSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.GPU_TYPE, Vendor: "THEAD", Model: "PPU-ZW810E"},
+			},
+		},
+	}
+	if !HasTHeadDevices(nil, normalizedSku) {
+		t.Fatal("expected normalized GPU+THEAD vendor sku to be detected as T-Head")
+	}
+}
+
+func TestHasKunlunxinDevices(t *testing.T) {
+	kunlunSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU},
+			},
+		},
+	}
+	if !HasKunlunxinDevices(nil, kunlunSku) {
+		t.Fatal("expected Kunlunxin XPU sku to be detected")
+	}
+
+	nvSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_NVIDIA_GPU},
+			},
+		},
+	}
+	if HasKunlunxinDevices(nil, nvSku) {
+		t.Fatal("expected NVIDIA sku not to be detected as Kunlunxin")
+	}
+
+	llm := &SLLM{
+		SLLMBase: SLLMBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU},
+			},
+		},
+	}
+	if !HasKunlunxinDevices(llm, nvSku) {
+		t.Fatal("expected llm device override to win over sku")
+	}
+
+	normalizedSku := &SLLMSku{
+		SLLMSkuBase: SLLMSkuBase{
+			Devices: &api.Devices{
+				{DevType: computeapi.GPU_TYPE, Vendor: "KUNLUNXIN", Model: "P800 OAM"},
+			},
+		},
+	}
+	if !HasKunlunxinDevices(nil, normalizedSku) {
+		t.Fatal("expected normalized GPU+KUNLUNXIN vendor sku to be detected as Kunlunxin")
+	}
+}

pkg/llm/models/sku.go

@@ -86,7 +86,7 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred
 	return input, nil
 }
 
-// normalizeLLMSkuDevices maps legacy NVIDIA_* / HYGON_* / ASCEND_* DevTypes onto
+// normalizeLLMSkuDevices maps NVIDIA_* / HYGON_* / ASCEND_* / ILUVATAR_* / THEAD_* / KUNLUNXIN_* DevTypes onto
 // GPU|NPU + SharingMode, and defaults empty DevType/SharingMode appropriately.
 func normalizeLLMSkuDevices(devices *api.Devices) error {
 	if devices == nil || len(*devices) == 0 {
@@ -160,6 +160,21 @@ func normalizeLLMSkuDevice(dev *api.Device) {
 		if dev.SharingMode == "" {
 			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
 		}
+	case computeapi.CONTAINER_DEV_ILUVATAR_GPU:
+		dev.DevType = computeapi.GPU_TYPE
+		if dev.SharingMode == "" {
+			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
+		}
+	case computeapi.CONTAINER_DEV_THEAD_PPU:
+		dev.DevType = computeapi.GPU_TYPE
+		if dev.SharingMode == "" {
+			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
+		}
+	case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU:
+		dev.DevType = computeapi.GPU_TYPE
+		if dev.SharingMode == "" {
+			dev.SharingMode = computeapi.DEVICE_SHARING_MODE_EXCLUSIVE
+		}
 	}
 	if dev.SharingMode == "" {
 		dev.SharingMode = computeapi.DEVICE_SHARING_MODE_HAMI
@@ -173,6 +188,12 @@ func normalizeLLMSkuDevice(dev *api.Device) {
 			dev.Vendor = "NVIDIA"
 		case computeapi.CONTAINER_DEV_ASCEND_NPU, computeapi.CONTAINER_DEV_ASCEND_NPU_HAMI:
 			dev.Vendor = "ASCEND"
+		case computeapi.CONTAINER_DEV_ILUVATAR_GPU:
+			dev.Vendor = "ILUVATAR"
+		case computeapi.CONTAINER_DEV_THEAD_PPU:
+			dev.Vendor = "THEAD"
+		case computeapi.CONTAINER_DEV_KUNLUNXIN_XPU:
+			dev.Vendor = "KUNLUNXIN"
 		}
 	}
 	dev.Vendor = canonicalizeLLMDeviceVendor(dev.Vendor)

pkg/llm/models/sku_hami_test.go

@@ -83,6 +83,34 @@ func TestNormalizeLLMSkuDeviceLegacyTypes(t *testing.T) {
 			wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
 			wantVendor:      "ASCEND",
 		},
+		{
+			name:            "ILUVATAR_GPU",
+			in:              api.Device{DevType: computeapi.CONTAINER_DEV_ILUVATAR_GPU},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
+			wantVendor:      "ILUVATAR",
+		},
+		{
+			name:            "THEAD_PPU",
+			in:              api.Device{DevType: computeapi.CONTAINER_DEV_THEAD_PPU},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
+			wantVendor:      "THEAD",
+		},
+		{
+			name:            "KUNLUNXIN_XPU",
+			in:              api.Device{DevType: computeapi.CONTAINER_DEV_KUNLUNXIN_XPU},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE,
+			wantVendor:      "KUNLUNXIN",
+		},
+		{
+			name:            "ILUVATAR vendor preserved and canonicalized",
+			in:              api.Device{DevType: computeapi.GPU_TYPE, Vendor: "iluvatar", Model: "BI-V150S"},
+			wantDevType:     computeapi.GPU_TYPE,
+			wantSharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
+			wantVendor:      "ILUVATAR",
+		},
 		{
 			name:            "ASCEND vendor empty DevType defaults to NPU",
 			in:              api.Device{Vendor: "ASCEND", Model: "910B2"},

pkg/monitor/dbinit/measurements/all.go

@@ -32,6 +32,8 @@ var All = []SMeasurement{
 	vasmi,
 	npuSmi,
 	hysmi,
+	ixsmi,
+	ppusmi,
 
 	worker,
 	serviceHttpCode,

pkg/monitor/dbinit/measurements/ixsmi.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var ixsmi = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"ixsmi", "Iluvatar GPU metrics",
+			monitor.METRIC_RES_TYPE_HOST, monitor.METRIC_DATABASE_TELE,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"temperature_gpu", "GPU temperature", "",
+		},
+		{
+			"temperature_memory", "GPU memory temperature", "",
+		},
+		{
+			"memory_total", "GPU memory total size", "",
+		},
+		{
+			"memory_free", "GPU memory free size", "",
+		},
+		{
+			"memory_used", "GPU memory used size", "",
+		},
+		{
+			"utilization_gpu", "GPU utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"utilization_memory", "GPU memory utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"power_draw", "GPU power draw", "",
+		},
+		{
+			"clocks_current_sm", "GPU current SM clocks, MHz", "",
+		},
+		{
+			"clocks_current_memory", "GPU current memory clocks, MHz", "",
+		},
+	},
+}

pkg/monitor/dbinit/measurements/metrics.csv

@@ -264,6 +264,26 @@ Measurement,MeasurementNote,ResourceType,Database,Metric,MetricNote,MetricUnit
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_memory","DCU memory utilization","%"
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_encoder","DCU encoder utilization","%"
 "hysmi","Hygon DCU metrics","host","telegraf","utilization_decoder","DCU decoder utilization","%"
+"ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_gpu","GPU temperature",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","temperature_memory","GPU memory temperature",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_total","GPU memory total size",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_free","GPU memory free size",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","memory_used","GPU memory used size",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","utilization_gpu","GPU utilization","%"
+"ixsmi","Iluvatar GPU metrics","host","telegraf","utilization_memory","GPU memory utilization","%"
+"ixsmi","Iluvatar GPU metrics","host","telegraf","power_draw","GPU power draw",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_sm","GPU current SM clocks, MHz",""
+"ixsmi","Iluvatar GPU metrics","host","telegraf","clocks_current_memory","GPU current memory clocks, MHz",""
+"ppusmi","T-Head PPU metrics","host","telegraf","temperature_gpu","PPU temperature",""
+"ppusmi","T-Head PPU metrics","host","telegraf","temperature_memory","PPU memory temperature",""
+"ppusmi","T-Head PPU metrics","host","telegraf","memory_total","PPU memory total size",""
+"ppusmi","T-Head PPU metrics","host","telegraf","memory_free","PPU memory free size",""
+"ppusmi","T-Head PPU metrics","host","telegraf","memory_used","PPU memory used size",""
+"ppusmi","T-Head PPU metrics","host","telegraf","utilization_gpu","PPU utilization","%"
+"ppusmi","T-Head PPU metrics","host","telegraf","utilization_memory","PPU memory utilization","%"
+"ppusmi","T-Head PPU metrics","host","telegraf","power_draw","PPU power draw",""
+"ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_sm","PPU current CU clocks, MHz",""
+"ppusmi","T-Head PPU metrics","host","telegraf","clocks_current_memory","PPU current memory clocks, MHz",""
 "worker","Worker queue","system","system","active_worker_cnt","Active Worker Count","NULL"
 "worker","Worker queue","system","system","max_worker_count","Max Worker Count","NULL"
 "worker","Worker queue","system","system","detach_worker_cnt","Detach worker Count","NULL"

pkg/monitor/dbinit/measurements/ppusmi.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package measurements
+
+import "yunion.io/x/onecloud/pkg/apis/monitor"
+
+var ppusmi = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"ppusmi", "T-Head PPU metrics",
+			monitor.METRIC_RES_TYPE_HOST, monitor.METRIC_DATABASE_TELE,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"temperature_gpu", "PPU temperature", "",
+		},
+		{
+			"temperature_memory", "PPU memory temperature", "",
+		},
+		{
+			"memory_total", "PPU memory total size", "",
+		},
+		{
+			"memory_free", "PPU memory free size", "",
+		},
+		{
+			"memory_used", "PPU memory used size", "",
+		},
+		{
+			"utilization_gpu", "PPU utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"utilization_memory", "PPU memory utilization", monitor.METRIC_UNIT_PERCENT,
+		},
+		{
+			"power_draw", "PPU power draw", "",
+		},
+		{
+			"clocks_current_sm", "PPU current CU clocks, MHz", "",
+		},
+		{
+			"clocks_current_memory", "PPU current memory clocks, MHz", "",
+		},
+	},
+}