mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git
synced 2026-09-20 08:03:53 +08:00
fix(host): mount kunlunxin xpu-smi and ml lib for containers (#25692)
Ensure container runtime mounts xpu-smi and libxpunvidia-ml.so.1 (including symlink targets) so XPU tooling works inside pods.
This commit is contained in:
@@ -78,7 +78,13 @@ func (m *kunlunxinXPUManager) NewContainerDevices(input *hostapi.ContainerCreate
|
|||||||
|
|
||||||
func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
func (m *kunlunxinXPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||||
indices := collectKunlunxinXpuVisibleIndices(devs)
|
indices := collectKunlunxinXpuVisibleIndices(devs)
|
||||||
return buildKunlunxinXpuExtraConfigures(indices, kunlunxinXreHome(), hygonPathExists)
|
return buildKunlunxinXpuExtraConfigures(
|
||||||
|
indices,
|
||||||
|
kunlunxinXreHome(),
|
||||||
|
kunlunxinXpuSmiPath(),
|
||||||
|
hygonPathExists,
|
||||||
|
procutils.RemoteReadlink,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
type kunlunxinXPU struct {
|
type kunlunxinXPU struct {
|
||||||
|
|||||||
@@ -22,12 +22,15 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||||
|
|
||||||
|
"yunion.io/x/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultKunlunxinXreHome = "/usr/local/xpu"
|
defaultKunlunxinXreHome = "/usr/local/xpu"
|
||||||
defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi"
|
defaultKunlunxinXpuSmiPath = "/usr/local/bin/xpu-smi"
|
||||||
kunlunxinXpuDevicePrefix = "/dev/xpu"
|
kunlunxinXpuDevicePrefix = "/dev/xpu"
|
||||||
|
kunlunxinXpuMlLibName = "libxpunvidia-ml.so.1"
|
||||||
)
|
)
|
||||||
|
|
||||||
var kunlunxinXpuCommonDevicePaths = []string{
|
var kunlunxinXpuCommonDevicePaths = []string{
|
||||||
@@ -186,23 +189,99 @@ func buildKunlunxinXpuRuntimeEnvs(indices []string, xreHome string, pathExists f
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildKunlunxinXpuRuntimeMounts(xreHome string, pathExists func(string) bool) []*runtimeapi.Mount {
|
func kunlunxinXpuMlLibPathCandidates(xreHome string) []string {
|
||||||
|
xreHome = normalizeKunlunxinXreHome(xreHome)
|
||||||
|
return []string{
|
||||||
|
path.Join("/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
|
||||||
|
path.Join("/usr/lib/x86_64-linux-gnu", kunlunxinXpuMlLibName),
|
||||||
|
path.Join(xreHome, "so", kunlunxinXpuMlLibName),
|
||||||
|
path.Join(xreHome, "lib64", kunlunxinXpuMlLibName),
|
||||||
|
path.Join(xreHome, "lib", kunlunxinXpuMlLibName),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveKunlunxinXpuMlLibPath(xreHome string, pathExists func(string) bool) string {
|
||||||
|
if pathExists == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, p := range kunlunxinXpuMlLibPathCandidates(xreHome) {
|
||||||
|
if pathExists(p) {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectSymlinkMountPaths returns start plus the final path from readlink
|
||||||
|
// (RemoteReadlink uses `readlink -f`, which resolves to the canonical target in one call).
|
||||||
|
func collectSymlinkMountPaths(start string, readlink func(string) (string, error)) []string {
|
||||||
|
if start == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
start = path.Clean(start)
|
||||||
|
out := []string{start}
|
||||||
|
if readlink == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
final, err := readlink(start)
|
||||||
|
if err != nil || final == "" {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
final = path.Clean(final)
|
||||||
|
if final == start {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
return append(out, final)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendReadonlyMountIfExists(mounts []*runtimeapi.Mount, hostPath string, pathExists func(string) bool, seen map[string]bool) []*runtimeapi.Mount {
|
||||||
|
if hostPath == "" || pathExists == nil || !pathExists(hostPath) || seen[hostPath] {
|
||||||
|
return mounts
|
||||||
|
}
|
||||||
|
seen[hostPath] = true
|
||||||
|
return append(mounts, &runtimeapi.Mount{
|
||||||
|
ContainerPath: hostPath,
|
||||||
|
HostPath: hostPath,
|
||||||
|
Readonly: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildKunlunxinXpuRuntimeMounts(
|
||||||
|
xreHome, smiPath string,
|
||||||
|
pathExists func(string) bool,
|
||||||
|
readlink func(string) (string, error),
|
||||||
|
) []*runtimeapi.Mount {
|
||||||
xreHome = normalizeKunlunxinXreHome(xreHome)
|
xreHome = normalizeKunlunxinXreHome(xreHome)
|
||||||
if pathExists == nil || !pathExists(xreHome) {
|
if pathExists == nil || !pathExists(xreHome) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return []*runtimeapi.Mount{
|
seen := map[string]bool{}
|
||||||
{
|
mounts := appendReadonlyMountIfExists(nil, xreHome, pathExists, seen)
|
||||||
ContainerPath: xreHome,
|
mounts = appendReadonlyMountIfExists(mounts, smiPath, pathExists, seen)
|
||||||
HostPath: xreHome,
|
|
||||||
Readonly: true,
|
mlPath := resolveKunlunxinXpuMlLibPath(xreHome, pathExists)
|
||||||
},
|
if mlPath == "" {
|
||||||
|
return mounts
|
||||||
}
|
}
|
||||||
|
for _, p := range collectSymlinkMountPaths(mlPath, readlink) {
|
||||||
|
if pathExists(p) {
|
||||||
|
mounts = appendReadonlyMountIfExists(mounts, p, pathExists, seen)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Warningf("kunlunxin xpu ml lib symlink target %s not found, skip mount", p)
|
||||||
|
}
|
||||||
|
return mounts
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildKunlunxinXpuExtraConfigures(indices []string, xreHome string, pathExists func(string) bool) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
func buildKunlunxinXpuExtraConfigures(
|
||||||
|
indices []string,
|
||||||
|
xreHome, smiPath string,
|
||||||
|
pathExists func(string) bool,
|
||||||
|
readlink func(string) (string, error),
|
||||||
|
) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) {
|
||||||
if len(indices) == 0 {
|
if len(indices) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists), buildKunlunxinXpuRuntimeMounts(xreHome, pathExists)
|
return buildKunlunxinXpuRuntimeEnvs(indices, xreHome, pathExists),
|
||||||
|
buildKunlunxinXpuRuntimeMounts(xreHome, smiPath, pathExists, readlink)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
package container_device
|
package container_device
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -78,7 +79,7 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
|
|||||||
exists := func(p string) bool {
|
exists := func(p string) bool {
|
||||||
return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib"
|
return p == "/usr/local/xpu" || p == "/usr/local/xpu/lib"
|
||||||
}
|
}
|
||||||
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", exists)
|
envs, mounts := buildKunlunxinXpuExtraConfigures([]string{"0", "3"}, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
|
||||||
require.Len(t, envs, 2)
|
require.Len(t, envs, 2)
|
||||||
assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key)
|
assert.Equal(t, "XPU_VISIBLE_DEVICES", envs[0].Key)
|
||||||
assert.Equal(t, "0,3", envs[0].Value)
|
assert.Equal(t, "0,3", envs[0].Value)
|
||||||
@@ -88,11 +89,76 @@ func TestBuildKunlunxinXpuExtraConfigures(t *testing.T) {
|
|||||||
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||||
assert.True(t, mounts[0].Readonly)
|
assert.True(t, mounts[0].Readonly)
|
||||||
|
|
||||||
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", exists)
|
envs, mounts = buildKunlunxinXpuExtraConfigures(nil, "/usr/local/xpu", "/usr/local/bin/xpu-smi", exists, nil)
|
||||||
assert.Nil(t, envs)
|
assert.Nil(t, envs)
|
||||||
assert.Nil(t, mounts)
|
assert.Nil(t, mounts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildKunlunxinXpuRuntimeMountsWithSmiAndMlLib(t *testing.T) {
|
||||||
|
smi := "/usr/local/bin/xpu-smi"
|
||||||
|
ml := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||||
|
exists := func(p string) bool {
|
||||||
|
switch p {
|
||||||
|
case "/usr/local/xpu", "/usr/local/xpu/lib", smi, ml:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, nil)
|
||||||
|
require.Len(t, mounts, 3)
|
||||||
|
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||||
|
assert.Equal(t, smi, mounts[1].HostPath)
|
||||||
|
assert.Equal(t, ml, mounts[2].HostPath)
|
||||||
|
for _, m := range mounts {
|
||||||
|
assert.Equal(t, m.HostPath, m.ContainerPath)
|
||||||
|
assert.True(t, m.Readonly)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectSymlinkMountPathsReadlinkF(t *testing.T) {
|
||||||
|
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||||
|
final := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
|
||||||
|
readlink := func(p string) (string, error) {
|
||||||
|
if p == link {
|
||||||
|
return final, nil // RemoteReadlink: readlink -f
|
||||||
|
}
|
||||||
|
return "", os.ErrInvalid
|
||||||
|
}
|
||||||
|
paths := collectSymlinkMountPaths(link, readlink)
|
||||||
|
assert.Equal(t, []string{link, final}, paths)
|
||||||
|
|
||||||
|
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, nil))
|
||||||
|
same := func(p string) (string, error) { return p, nil }
|
||||||
|
assert.Equal(t, []string{link}, collectSymlinkMountPaths(link, same))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildKunlunxinXpuRuntimeMountsFollowsSymlink(t *testing.T) {
|
||||||
|
smi := "/usr/local/bin/xpu-smi"
|
||||||
|
link := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1"
|
||||||
|
real := "/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1.0.0"
|
||||||
|
exists := func(p string) bool {
|
||||||
|
switch p {
|
||||||
|
case "/usr/local/xpu", smi, link, real:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
readlink := func(p string) (string, error) {
|
||||||
|
if p == link {
|
||||||
|
return real, nil
|
||||||
|
}
|
||||||
|
return "", os.ErrInvalid
|
||||||
|
}
|
||||||
|
mounts := buildKunlunxinXpuRuntimeMounts("/usr/local/xpu", smi, exists, readlink)
|
||||||
|
require.Len(t, mounts, 4)
|
||||||
|
assert.Equal(t, "/usr/local/xpu", mounts[0].HostPath)
|
||||||
|
assert.Equal(t, smi, mounts[1].HostPath)
|
||||||
|
assert.Equal(t, link, mounts[2].HostPath)
|
||||||
|
assert.Equal(t, real, mounts[3].HostPath)
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseKunlunxinXpuNodeIndex(t *testing.T) {
|
func TestParseKunlunxinXpuNodeIndex(t *testing.T) {
|
||||||
idx, ok := parseKunlunxinXpuNodeIndex("xpu3")
|
idx, ok := parseKunlunxinXpuNodeIndex("xpu3")
|
||||||
assert.True(t, ok)
|
assert.True(t, ok)
|
||||||
|
|||||||
Reference in New Issue
Block a user