mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git
synced 2026-09-20 08:03:53 +08:00
fix(host): mount kunlunxin xpu-smi and ml lib for containers (#25693)
Ensure container runtime mounts xpu-smi and libxpunvidia-ml.so.1 (including symlink targets) so XPU tooling works inside pods.
This commit is contained in:
@@ -78,7 +78,13 @@ func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreate
|
||||
|
||||
func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
indices := collectKunlunxinXpuVisibleIndices(devs)
|
||||
return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists)
|
||||
return buildKunlunxinXpuExtraConfigures(
|
||||
indices,
|
||||
kunlunxinXreHome(),
|
||||
kunlunxinXpuSmiPath(),
|
||||
hygonPathExists,
|
||||
procutils.RemoteReadlink,
|
||||
)
|
||||
}
|
||||
|
||||
type kunlunxinXPU struct {
|
||||
|
||||
@@ -22,12 +22,15 @@ import (
|
||||
"strings"
|
||||
|
||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||
|
||||
"yunion.io/x/log"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultKunlunxinXreHome = "/usr/local/xpu"
|
||||
defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi"
|
||||
kunlunxinXpuDevicePrefix = "/dev/xpu"
|
||||
kunlunxinXpuMlLibName = "libxpunvidia-ml.so.1"
|
||||
)
|
||||
|
||||
var kunlunxinXpuCommonDevicePaths = []string{
|
||||
@@ -186,23 +189,99 @@ func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists f
|
||||
}
|
||||
}
|
||||
|
||||
func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount {
|
||||
func kunlunxinXpuMlLibPathCandidates(xreHome string) []string {
|
||||
xreHome = normalizeKunlunxinXreHome(xreHome)
|
||||
return []string{
|
||||
path.Join("/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
|
||||
path.Join("/usr/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
|
||||
path.Join(xreHome, "so", kunlunxinXpuMlLibName),
|
||||
path.Join(xreHome, "lib64", kunlunxinXpuMlLibName),
|
||||
path.Join(xreHome, "lib", kunlunxinXpuMlLibName),
|
||||
}
|
||||
}
|
||||
|
||||
func resolveKunlunxinXpuMlLibPath(xreHome string, pathExists func(string) bool) string {
|
||||
if pathExists == nil {
|
||||
return ""
|
||||
}
|
||||
for _, p := range kunlunxinXpuMlLibPathCandidates(xreHome) {
|
||||
if pathExists(p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// collectSymlinkMountPaths returns start plus the final path from readlink
|
||||
// (RemoteReadlink uses `readlink -f`, which resolves to the canonical target in one call).
|
||||
func collectSymlinkMountPaths(start string, readlink func(string) (string, error)) []string {
|
||||
if start == "" {
|
||||
return nil
|
||||
}
|
||||
start = path.Clean(start)
|
||||
out := []string{start}
|
||||
if readlink == nil {
|
||||
return out
|
||||
}
|
||||
final, err := readlink(start)
|
||||
if err != nil || final == "" {
|
||||
return out
|
||||
}
|
||||
final = path.Clean(final)
|
||||
if final == start {
|
||||
return out
|
||||
}
|
||||
return append(out, final)
|
||||
}
|
||||
|
||||
func appendReadonlyMountIfExists(mounts []*runtimeapi.Mount, hostPath string, pathExists func(string) bool, seen map[string]bool) []*runtimeapi.Mount {
|
||||
if hostPath == "" || pathExists == nil || !pathExists(hostPath) || seen[hostPath] {
|
||||
return mounts
|
||||
}
|
||||
seen[hostPath] = true
|
||||
return append(mounts, &runtimeapi.Mount{
|
||||
ContainerPath: hostPath,
|
||||
HostPath: hostPath,
|
||||
Readonly: true,
|
||||
})
|
||||
}
|
||||
|
||||
func buildKunlunxinXpuRuntimeMounts(
|
||||
xreHome, smiPath string,
|
||||
pathExists func(string) bool,
|
||||
readlink func(string) (string, error),
|
||||
) []*runtimeapi.Mount {
|
||||
xreHome = normalizeKunlunxinXreHome(xreHome)
|
||||
if pathExists == nil || !pathExists(xreHome) {
|
||||
return nil
|
||||
}
|
||||
return []*runtimeapi.Mount{
|
||||
{
|
||||
ContainerPath: xreHome,
|
||||
HostPath: xreHome,
|
||||
Readonly: true,
|
||||
},
|
||||
seen := map[string]bool{}
|
||||
mounts := appendReadonlyMountIfExists(nil, xreHome, pathExists, seen)
|
||||
mounts = appendReadonlyMountIfExists(mounts, smiPath, pathExists, seen)
|
||||
|
||||
mlPath := resolveKunlunxinXpuMlLibPath(xreHome, pathExists)
|
||||
if mlPath == "" {
|
||||
return mounts
|
||||
}
|
||||
for _, p := range collectSymlinkMountPaths(mlPath, readlink) {
|
||||
if pathExists(p) {
|
||||
mounts = appendReadonlyMountIfExists(mounts, p, pathExists, seen)
|
||||
continue
|
||||
}
|
||||
log.Warningf("kunlunxin xpu ml lib symlink target %s not found, skip mount", p)
|
||||
}
|
||||
return mounts
|
||||
}
|
||||
|
||||
func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
func buildKunlunxinXpuExtraConfigures(
|
||||
indices []string,
|
||||
xreHome, smiPath string,
|
||||
pathExists func(string) bool,
|
||||
readlink func(string) (string, error),
|
||||
) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||
if len(indices) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists)
|
||||
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists),
|
||||
buildKunlunxinXpuRuntimeMounts(xreHome, smiPath, pathExists, readlink)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
package container_device
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -78,7 +79,7 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
|
||||
exists := func(p string) bool {
|
||||
return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib"
|
||||
}
|
||||
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists)
|
||||
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
|
||||
require.Len(t, envs, 2)
|
||||
assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key)
|
||||
assert.Equal(t, "0,3", envs[0].Value)
|
||||
@@ -88,11 +89,76 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
|
||||
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||
assert.True(t, mounts[0].Readonly)
|
||||
|
||||
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists)
|
||||
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
|
||||
assert.Nil(t, envs)
|
||||
assert.Nil(t, mounts)
|
||||
}
|
||||
|
||||
func TestBuildKunlunxinXpuRuntimeMountsWithSmiAndMlLib(t *testing.T) {
|
||||
smi := "/usr/local/bin/xpu-smi"
|
||||
ml := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||
exists := func(p string) bool {
|
||||
switch p {
|
||||
case "/usr/local/xpu", "/usr/local/xpu/lib", smi, ml:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, nil)
|
||||
require.Len(t, mounts, 3)
|
||||
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||
assert.Equal(t, smi, mounts[1].HostPath)
|
||||
assert.Equal(t, ml, mounts[2].HostPath)
|
||||
for _, m := range mounts {
|
||||
assert.Equal(t, m.HostPath, m.ContainerPath)
|
||||
assert.True(t, m.Readonly)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectSymlinkMountPathsReadlinkF(t *testing.T) {
|
||||
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||
final := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
|
||||
readlink := func(p string) (string, error) {
|
||||
if p == link {
|
||||
return final, nil // RemoteReadlink: readlink -f
|
||||
}
|
||||
return "", os.ErrInvalid
|
||||
}
|
||||
paths := collectSymlinkMountPaths(link, readlink)
|
||||
assert.Equal(t, []string{link, final}, paths)
|
||||
|
||||
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, nil))
|
||||
same := func(p string) (string, error) { return p, nil }
|
||||
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, same))
|
||||
}
|
||||
|
||||
func TestBuildKunlunxinXpuRuntimeMountsFollowsSymlink(t *testing.T) {
|
||||
smi := "/usr/local/bin/xpu-smi"
|
||||
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||
real := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
|
||||
exists := func(p string) bool {
|
||||
switch p {
|
||||
case "/usr/local/xpu", smi, link, real:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
readlink := func(p string) (string, error) {
|
||||
if p == link {
|
||||
return real, nil
|
||||
}
|
||||
return "", os.ErrInvalid
|
||||
}
|
||||
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, readlink)
|
||||
require.Len(t, mounts, 4)
|
||||
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||
assert.Equal(t, smi, mounts[1].HostPath)
|
||||
assert.Equal(t, link, mounts[2].HostPath)
|
||||
assert.Equal(t, real, mounts[3].HostPath)
|
||||
}
|
||||
|
||||
func TestParseKunlunxinXpuNodeIndex(t *testing.T) {
|
||||
idx, ok := parseKunlunxinXpuNodeIndex("xpu3")
|
||||
assert.True(t, ok)
|
||||
|
||||
Reference in New Issue
Block a user