diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go index 761e393db6..9befd4bb56 100644 --- a/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpu.go @@ -78,7 +78,13 @@ func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreate func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { indices := collectKunlunxinXpuVisibleIndices(devs) - return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists) + return buildKunlunxinXpuExtraConfigures( + indices, + kunlunxinXreHome(), + kunlunxinXpuSmiPath(), + hygonPathExists, + procutils.RemoteReadlink, + ) } type kunlunxinXPU struct { diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go index 196fdc0fd5..6c7e37ba12 100644 --- a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi.go @@ -22,12 +22,15 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" ) const ( defaultKunlunxinXreHome = "/usr/local/xpu" defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi" kunlunxinXpuDevicePrefix = "/dev/xpu" + kunlunxinXpuMlLibName = "libxpunvidia-ml.so.1" ) var kunlunxinXpuCommonDevicePaths = []string{ @@ -186,23 +189,99 @@ func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists f } } -func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount { +func kunlunxinXpuMlLibPathCandidates(xreHome string) []string { + xreHome = normalizeKunlunxinXreHome(xreHome) + return []string{ + path.Join("/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName), + path.Join("/usr/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName), + path.Join(xreHome, "so", kunlunxinXpuMlLibName), + path.Join(xreHome, "lib64", kunlunxinXpuMlLibName), + path.Join(xreHome, "lib", kunlunxinXpuMlLibName), + } +} + +func resolveKunlunxinXpuMlLibPath(xreHome string, pathExists func(string) bool) string { + if pathExists == nil { + return "" + } + for _, p := range kunlunxinXpuMlLibPathCandidates(xreHome) { + if pathExists(p) { + return p + } + } + return "" +} + +// collectSymlinkMountPaths returns start plus the final path from readlink +// (RemoteReadlink uses `readlink -f`, which resolves to the canonical target in one call). +func collectSymlinkMountPaths(start string, readlink func(string) (string, error)) []string { + if start == "" { + return nil + } + start = path.Clean(start) + out := []string{start} + if readlink == nil { + return out + } + final, err := readlink(start) + if err != nil || final == "" { + return out + } + final = path.Clean(final) + if final == start { + return out + } + return append(out, final) +} + +func appendReadonlyMountIfExists(mounts []*runtimeapi.Mount, hostPath string, pathExists func(string) bool, seen map[string]bool) []*runtimeapi.Mount { + if hostPath == "" || pathExists == nil || !pathExists(hostPath) || seen[hostPath] { + return mounts + } + seen[hostPath] = true + return append(mounts, &runtimeapi.Mount{ + ContainerPath: hostPath, + HostPath: hostPath, + Readonly: true, + }) +} + +func buildKunlunxinXpuRuntimeMounts( + xreHome, smiPath string, + pathExists func(string) bool, + readlink func(string) (string, error), +) []*runtimeapi.Mount { xreHome = normalizeKunlunxinXreHome(xreHome) if pathExists == nil || !pathExists(xreHome) { return nil } - return []*runtimeapi.Mount{ - { - ContainerPath: xreHome, - HostPath: xreHome, - Readonly: true, - }, + seen := map[string]bool{} + mounts := appendReadonlyMountIfExists(nil, xreHome, pathExists, seen) + mounts = appendReadonlyMountIfExists(mounts, smiPath, pathExists, seen) + + mlPath := resolveKunlunxinXpuMlLibPath(xreHome, pathExists) + if mlPath == "" { + return mounts } + for _, p := range collectSymlinkMountPaths(mlPath, readlink) { + if pathExists(p) { + mounts = appendReadonlyMountIfExists(mounts, p, pathExists, seen) + continue + } + log.Warningf("kunlunxin xpu ml lib symlink target %s not found, skip mount", p) + } + return mounts } -func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { +func buildKunlunxinXpuExtraConfigures( + indices []string, + xreHome, smiPath string, + pathExists func(string) bool, + readlink func(string) (string, error), +) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { if len(indices) == 0 { return nil, nil } - return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists) + return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), + buildKunlunxinXpuRuntimeMounts(xreHome, smiPath, pathExists, readlink) } diff --git a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go index dbbe0b4473..45a0731ce9 100644 --- a/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go +++ b/pkg/hostman/isolated_device/container_device/kunlunxin_xpusmi_test.go @@ -15,6 +15,7 @@ package container_device import ( + "os" "testing" "github.com/stretchr/testify/assert" @@ -78,7 +79,7 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) { exists := func(p string) bool { return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib" } - envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists) + envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil) require.Len(t, envs, 2) assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key) assert.Equal(t, "0,3", envs[0].Value) @@ -88,11 +89,76 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) { assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath) assert.True(t, mounts[0].Readonly) - envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists) + envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil) assert.Nil(t, envs) assert.Nil(t, mounts) } +func TestBuildKunlunxinXpuRuntimeMountsWithSmiAndMlLib(t *testing.T) { + smi := "/usr/local/bin/xpu-smi" + ml := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1" + exists := func(p string) bool { + switch p { + case "/usr/local/xpu", "/usr/local/xpu/lib", smi, ml: + return true + default: + return false + } + } + mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, nil) + require.Len(t, mounts, 3) + assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath) + assert.Equal(t, smi, mounts[1].HostPath) + assert.Equal(t, ml, mounts[2].HostPath) + for _, m := range mounts { + assert.Equal(t, m.HostPath, m.ContainerPath) + assert.True(t, m.Readonly) + } +} + +func TestCollectSymlinkMountPathsReadlinkF(t *testing.T) { + link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1" + final := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0" + readlink := func(p string) (string, error) { + if p == link { + return final, nil // RemoteReadlink: readlink -f + } + return "", os.ErrInvalid + } + paths := collectSymlinkMountPaths(link, readlink) + assert.Equal(t, []string{link, final}, paths) + + assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, nil)) + same := func(p string) (string, error) { return p, nil } + assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, same)) +} + +func TestBuildKunlunxinXpuRuntimeMountsFollowsSymlink(t *testing.T) { + smi := "/usr/local/bin/xpu-smi" + link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1" + real := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0" + exists := func(p string) bool { + switch p { + case "/usr/local/xpu", smi, link, real: + return true + default: + return false + } + } + readlink := func(p string) (string, error) { + if p == link { + return real, nil + } + return "", os.ErrInvalid + } + mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, readlink) + require.Len(t, mounts, 4) + assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath) + assert.Equal(t, smi, mounts[1].HostPath) + assert.Equal(t, link, mounts[2].HostPath) + assert.Equal(t, real, mounts[3].HostPath) +} + func TestParseKunlunxinXpuNodeIndex(t *testing.T) { idx, ok := parseKunlunxinXpuNodeIndex("xpu3") assert.True(t, ok)