fix(host): mount kunlunxin xpu-smi and ml lib for containers (#25692)

Ensure container runtime mounts xpu-smi and libxpunvidia-ml.so.1
(including symlink targets) so XPU tooling works inside pods.
This commit is contained in:
Zexi Li
2026-09-18 10:57:54 +08:00
committed by GitHub
parent 8af361db41
commit 4130e94f4e
3 changed files with 163 additions and 12 deletions

View File

@@ -78,7 +78,13 @@ func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreate
func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
indices := collectKunlunxinXpuVisibleIndices(devs)
return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists)
return buildKunlunxinXpuExtraConfigures(
indices,
kunlunxinXreHome(),
kunlunxinXpuSmiPath(),
hygonPathExists,
procutils.RemoteReadlink,
)
}
type kunlunxinXPU struct {

View File

@@ -22,12 +22,15 @@ import (
"strings"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
"yunion.io/x/log"
)
const (
defaultKunlunxinXreHome = "/usr/local/xpu"
defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi"
kunlunxinXpuDevicePrefix = "/dev/xpu"
kunlunxinXpuMlLibName = "libxpunvidia-ml.so.1"
)
var kunlunxinXpuCommonDevicePaths = []string{
@@ -186,23 +189,99 @@ func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists f
}
}
func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount {
func kunlunxinXpuMlLibPathCandidates(xreHome string) []string {
xreHome = normalizeKunlunxinXreHome(xreHome)
return []string{
path.Join("/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
path.Join("/usr/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
path.Join(xreHome, "so", kunlunxinXpuMlLibName),
path.Join(xreHome, "lib64", kunlunxinXpuMlLibName),
path.Join(xreHome, "lib", kunlunxinXpuMlLibName),
}
}
func resolveKunlunxinXpuMlLibPath(xreHome string, pathExists func(string) bool) string {
if pathExists == nil {
return ""
}
for _, p := range kunlunxinXpuMlLibPathCandidates(xreHome) {
if pathExists(p) {
return p
}
}
return ""
}
// collectSymlinkMountPaths returns start plus the final path from readlink
// (RemoteReadlink uses `readlink -f`, which resolves to the canonical target in one call).
func collectSymlinkMountPaths(start string, readlink func(string) (string, error)) []string {
if start == "" {
return nil
}
start = path.Clean(start)
out := []string{start}
if readlink == nil {
return out
}
final, err := readlink(start)
if err != nil || final == "" {
return out
}
final = path.Clean(final)
if final == start {
return out
}
return append(out, final)
}
func appendReadonlyMountIfExists(mounts []*runtimeapi.Mount, hostPath string, pathExists func(string) bool, seen map[string]bool) []*runtimeapi.Mount {
if hostPath == "" || pathExists == nil || !pathExists(hostPath) || seen[hostPath] {
return mounts
}
seen[hostPath] = true
return append(mounts, &runtimeapi.Mount{
ContainerPath: hostPath,
HostPath: hostPath,
Readonly: true,
})
}
func buildKunlunxinXpuRuntimeMounts(
xreHome, smiPath string,
pathExists func(string) bool,
readlink func(string) (string, error),
) []*runtimeapi.Mount {
xreHome = normalizeKunlunxinXreHome(xreHome)
if pathExists == nil || !pathExists(xreHome) {
return nil
}
return []*runtimeapi.Mount{
{
ContainerPath: xreHome,
HostPath: xreHome,
Readonly: true,
},
seen := map[string]bool{}
mounts := appendReadonlyMountIfExists(nil, xreHome, pathExists, seen)
mounts = appendReadonlyMountIfExists(mounts, smiPath, pathExists, seen)
mlPath := resolveKunlunxinXpuMlLibPath(xreHome, pathExists)
if mlPath == "" {
return mounts
}
for _, p := range collectSymlinkMountPaths(mlPath, readlink) {
if pathExists(p) {
mounts = appendReadonlyMountIfExists(mounts, p, pathExists, seen)
continue
}
log.Warningf("kunlunxin xpu ml lib symlink target %s not found, skip mount", p)
}
return mounts
}
func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
func buildKunlunxinXpuExtraConfigures(
indices []string,
xreHome, smiPath string,
pathExists func(string) bool,
readlink func(string) (string, error),
) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
if len(indices) == 0 {
return nil, nil
}
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists)
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists),
buildKunlunxinXpuRuntimeMounts(xreHome, smiPath, pathExists, readlink)
}

View File

@@ -15,6 +15,7 @@
package container_device
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
@@ -78,7 +79,7 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
exists := func(p string) bool {
return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib"
}
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists)
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
require.Len(t, envs, 2)
assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key)
assert.Equal(t, "0,3", envs[0].Value)
@@ -88,11 +89,76 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
assert.True(t, mounts[0].Readonly)
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists)
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
assert.Nil(t, envs)
assert.Nil(t, mounts)
}
func TestBuildKunlunxinXpuRuntimeMountsWithSmiAndMlLib(t *testing.T) {
smi := "/usr/local/bin/xpu-smi"
ml := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
exists := func(p string) bool {
switch p {
case "/usr/local/xpu", "/usr/local/xpu/lib", smi, ml:
return true
default:
return false
}
}
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, nil)
require.Len(t, mounts, 3)
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
assert.Equal(t, smi, mounts[1].HostPath)
assert.Equal(t, ml, mounts[2].HostPath)
for _, m := range mounts {
assert.Equal(t, m.HostPath, m.ContainerPath)
assert.True(t, m.Readonly)
}
}
func TestCollectSymlinkMountPathsReadlinkF(t *testing.T) {
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
final := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
readlink := func(p string) (string, error) {
if p == link {
return final, nil // RemoteReadlink: readlink -f
}
return "", os.ErrInvalid
}
paths := collectSymlinkMountPaths(link, readlink)
assert.Equal(t, []string{link, final}, paths)
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, nil))
same := func(p string) (string, error) { return p, nil }
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, same))
}
func TestBuildKunlunxinXpuRuntimeMountsFollowsSymlink(t *testing.T) {
smi := "/usr/local/bin/xpu-smi"
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
real := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
exists := func(p string) bool {
switch p {
case "/usr/local/xpu", smi, link, real:
return true
default:
return false
}
}
readlink := func(p string) (string, error) {
if p == link {
return real, nil
}
return "", os.ErrInvalid
}
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, readlink)
require.Len(t, mounts, 4)
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
assert.Equal(t, smi, mounts[1].HostPath)
assert.Equal(t, link, mounts[2].HostPath)
assert.Equal(t, real, mounts[3].HostPath)
}
func TestParseKunlunxinXpuNodeIndex(t *testing.T) {
idx, ok := parseKunlunxinXpuNodeIndex("xpu3")
assert.True(t, ok)