fix(region,scheduler,host): container numa aware (#21506)

- optimize host register isolated device update - container device base try detect numa node - container alloc numa memory - calc numa distance before select isolated device Signed-off-by: wanyaoqi <d3lx.yq@gmail.com>
2026-09-20 08:03:53 +08:00 · 2024-11-02 07:50:32 +08:00
parent 8d45dcf185
commit dc88d51d12
22 changed files with 522 additions and 103 deletions
--- a/pkg/apis/scheduler/api.go
+++ b/pkg/apis/scheduler/api.go
@@ -88,7 +88,8 @@ type ScheduleInput struct {
 	ResetCpuNumaPin bool   `json:"reset_cpu_numa_pin"`

 	// For Migrate
-	CpuNumaPin []SCpuNumaPin `json:"cpu_numa_pin"`
+	CpuNumaPin      []SCpuNumaPin `json:"cpu_numa_pin"`
+	PreferNumaNodes []int         `json:"prefer_numa_nodes"`

 	HostMemPageSizeKB int    `json:"host_mem_page_size"`
 	SkipKernelCheck   *bool  `json:"skip_kernel_check"`
--- a/pkg/baremetal/tasks/baseprepare.go
+++ b/pkg/baremetal/tasks/baseprepare.go
@@ -872,7 +872,7 @@ func (task *sBaremetalPrepareTask) sendIsolatedDevicesInfo(
 	}

 	for i := 0; i < len(gpuDevs); i++ {
-		if _, err := isolated_device.SyncDeviceInfo(session, task.baremetal.GetId(), gpuDevs[i]); err != nil {
+		if _, err := isolated_device.SyncDeviceInfo(session, task.baremetal.GetId(), gpuDevs[i], true); err != nil {
 			return errors.Wrap(err, "sync device info")
 		}
 	}
--- a/pkg/compute/models/guest_actions.go
+++ b/pkg/compute/models/guest_actions.go
@@ -1558,14 +1558,15 @@ func (self *SGuest) StartGueststartTask(
 	data *jsonutils.JSONDict, parentTaskId string,
 ) error {
 	schedStart := self.Hypervisor == api.HYPERVISOR_KVM && self.guestDisksStorageTypeIsShared()
+	startFromCreate := jsonutils.QueryBoolean(data, "start_from_create", false)
 	if options.Options.IgnoreNonrunningGuests {
 		host := HostManager.FetchHostById(self.HostId)
-		if host != nil && host.EnableNumaAllocate {
+		if !startFromCreate && host != nil && host.EnableNumaAllocate {
 			schedStart = true
 		}
 	}

-	if self.CpuNumaPin != nil {
+	if !startFromCreate && self.CpuNumaPin != nil {
 		// clean cpu numa pin
 		err := self.SetCpuNumaPin(ctx, userCred, nil, nil)
 		if err != nil {
--- a/pkg/compute/models/guest_queries.go
+++ b/pkg/compute/models/guest_queries.go
@@ -824,6 +824,7 @@ func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevi
 		dev.GuestId = devs[i].GuestId
 		dev.Addr = devs[i].Addr
 		dev.VendorDeviceId = devs[i].VendorDeviceId
+		dev.NumaNode = byte(devs[i].NumaNode)
 		gdevs, ok := ret[devs[i].GuestId]
 		if !ok {
 			gdevs = make([]api.SIsolatedDevice, 0)
--- a/pkg/compute/models/guests.go
+++ b/pkg/compute/models/guests.go
@@ -4322,7 +4322,7 @@ func (self *SGuest) allocSriovNicDevice(
 	}
 	netConfig.SriovDevice.NetworkIndex = &gn.Index
 	netConfig.SriovDevice.WireId = net.WireId
-	err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone, nil)
+	err = self.createIsolatedDeviceOnHost(ctx, userCred, host, netConfig.SriovDevice, pendingUsageZone, nil, nil)
 	if err != nil {
 		return errors.Wrap(err, "self.createIsolatedDeviceOnHost")
 	}
@@ -4511,7 +4511,7 @@ func (self *SGuest) attachNVMEDevice(
 ) error {
 	gd := self.GetGuestDisk(disk.Id)
 	diskConfig.NVMEDevice.DiskIndex = &gd.Index
-	err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage, nil)
+	err := self.createIsolatedDeviceOnHost(ctx, userCred, host, diskConfig.NVMEDevice, pendingUsage, nil, nil)
 	if err != nil {
 		return errors.Wrap(err, "self.createIsolatedDeviceOnHost")
 	}
@@ -4664,12 +4664,23 @@ func (self *SGuest) createDiskOnHost(
 }

 func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error {
+	var numaNodes []int
+	if self.CpuNumaPin != nil {
+		numaNodes = make([]int, 0)
+		cpuNumaPin := make([]schedapi.SCpuNumaPin, 0)
+		self.CpuNumaPin.Unmarshal(&cpuNumaPin)
+
+		for i := range cpuNumaPin {
+			numaNodes = append(numaNodes, cpuNumaPin[i].NodeId)
+		}
+	}
+
 	usedDeviceMap := map[string]*SIsolatedDevice{}
 	for _, devConfig := range devs {
 		if devConfig.DevType == api.NIC_TYPE || devConfig.DevType == api.NVME_PT_TYPE {
 			continue
 		}
-		err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage, usedDeviceMap)
+		err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage, usedDeviceMap, numaNodes)
 		if err != nil {
 			return err
 		}
@@ -4677,11 +4688,11 @@ func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcc
 	return nil
 }

-func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota, usedDevMap map[string]*SIsolatedDevice) error {
+func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int) error {
 	lockman.LockClass(ctx, QuotaManager, self.ProjectId)
 	defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId)

-	err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred, usedDevMap)
+	err := IsolatedDeviceManager.attachHostDeviceToGuestByDesc(ctx, self, host, devConfig, userCred, usedDevMap, preferNumaNodes)
 	if err != nil {
 		return err
 	}
--- a/pkg/compute/models/isolated_devices.go
+++ b/pkg/compute/models/isolated_devices.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"math"
 	"reflect"
 	"sort"
 	"strings"
@@ -33,6 +34,7 @@ import (
 	"yunion.io/x/sqlchemy"

 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -573,13 +575,16 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe
 	return nil
 }

-func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice) error {
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(
+	ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig,
+	userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int,
+) error {
 	if len(devConfig.Id) > 0 {
 		return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred)
 	} else if len(devConfig.DevicePath) > 0 {
-		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred, usedDevMap)
+		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred, usedDevMap, preferNumaNodes)
 	} else {
-		return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred, usedDevMap)
+		return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred, usedDevMap, preferNumaNodes)
 	}
 }

@@ -595,7 +600,7 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C
 	return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex)
 }

-func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice) error {
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int) error {
 	if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 {
 		return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig)
 	}
@@ -647,7 +652,10 @@ func (pq *SorttedGroupDevs) Pop() interface{} {
 	return item
 }

-func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice) error {
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(
+	ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig,
+	userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int,
+) error {
 	if len(devConfig.Model) == 0 {
 		return fmt.Errorf("Not found model from info: %#v", devConfig)
 	}
@@ -680,27 +688,66 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx contex
 	}
 	sort.Sort(groupDevs)

-	var preferNumaNode int8 = -1
-	for _, dev := range usedDevMap {
-		if dev.NumaNode >= 0 {
-			preferNumaNode = dev.NumaNode
-			break
-		}
-	}
-
 	var selectedDev *SIsolatedDevice
-	if preferNumaNode >= 0 {
-		for i := range groupDevs {
-			if groupDevs[i].DevPath == "" {
-				for j := range groupDevs[i].Devs {
-					if groupDevs[i].Devs[j].NumaNode == preferNumaNode {
-						selectedDev = &groupDevs[i].Devs[j]
-						break
+	if len(preferNumaNodes) > 0 {
+		topoObj, err := host.SysInfo.Get("topology")
+		if err != nil {
+			return errors.Wrap(err, "get topology from host sys_info")
+		}
+		hostTopo := new(hostapi.HostTopology)
+		if err := topoObj.Unmarshal(hostTopo); err != nil {
+			return errors.Wrap(err, "Unmarshal host topology struct")
+		}
+
+		if len(groupDevs) == 1 && groupDevs[0].DevPath == "" {
+			minDistancesDevIdx := -1
+			minDistances := math.MaxInt32
+			for i := range groupDevs[0].Devs {
+				if groupDevs[0].Devs[i].NumaNode < 0 {
+					continue
+				}
+				devNodeId := groupDevs[0].Devs[i].NumaNode
+				for j := range hostTopo.Nodes {
+					if hostTopo.Nodes[j].ID == int(devNodeId) {
+						devDistance := 0
+						for k := range preferNumaNodes {
+							devDistance += hostTopo.Nodes[j].Distances[preferNumaNodes[k]]
+						}
+						if devDistance < minDistances {
+							minDistances = devDistance
+							minDistancesDevIdx = i
+						}
 					}
 				}
-			} else if groupDevs[i].Devs[0].NumaNode == preferNumaNode {
-				selectedDev = &groupDevs[i].Devs[0]
-				break
+			}
+			if minDistancesDevIdx >= 0 {
+				selectedDev = &groupDevs[0].Devs[minDistancesDevIdx]
+			}
+		} else {
+			minDistancesGroupIdx := -1
+			minDistances := math.MaxInt32
+			log.Infof("devtype %s grouplength %d", groupDevs[0].Devs[0].DevType, len(groupDevs))
+
+			for i := range groupDevs {
+				if groupDevs[i].Devs[0].NumaNode < 0 {
+					continue
+				}
+				devNodeId := groupDevs[i].Devs[0].NumaNode
+				for j := range hostTopo.Nodes {
+					if hostTopo.Nodes[j].ID == int(devNodeId) {
+						devDistance := 0
+						for k := range preferNumaNodes {
+							devDistance += hostTopo.Nodes[j].Distances[preferNumaNodes[k]]
+						}
+						if devDistance < minDistances {
+							minDistances = devDistance
+							minDistancesGroupIdx = i
+						}
+					}
+				}
+			}
+			if minDistancesGroupIdx >= 0 {
+				selectedDev = &groupDevs[minDistancesGroupIdx].Devs[0]
 			}
 		}
 	}
--- a/pkg/compute/tasks/guest_create_task.go
+++ b/pkg/compute/tasks/guest_create_task.go
@@ -279,7 +279,9 @@ func (self *GuestCreateTask) OnDeployEipComplete(ctx context.Context, obj db.ISt

 	if jsonutils.QueryBoolean(self.GetParams(), "auto_start", false) {
 		self.SetStage("OnAutoStartGuest", nil)
-		guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetTaskId())
+		params := jsonutils.NewDict()
+		params.Set("start_from_create", jsonutils.JSONTrue)
+		guest.StartGueststartTask(ctx, self.GetUserCred(), params, self.GetTaskId())
 	} else {
 		self.SetStage("OnSyncStatusComplete", nil)
 		guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
--- a/pkg/compute/tasks/guest_live_migrate_task.go
+++ b/pkg/compute/tasks/guest_live_migrate_task.go
@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 )

@@ -87,7 +88,18 @@ func (task *GuestMigrateTask) GetSchedParams() (*schedapi.ScheduleInput, error)
 		input.SkipCpuCheck = skipCpuCheck
 		input.SkipKernelCheck = skipKernelCheck
 	}
-	return guest.GetSchedMigrateParams(task.GetUserCred(), input), nil
+	res := guest.GetSchedMigrateParams(task.GetUserCred(), input)
+
+	if devs, _ := guest.GetIsolatedDevices(); len(devs) > 0 {
+		preferNumaNodesSet := cpuset.NewBuilder()
+		for i := range devs {
+			if devs[i].NumaNode >= 0 {
+				preferNumaNodesSet.Add(int(devs[i].NumaNode))
+			}
+		}
+		res.PreferNumaNodes = preferNumaNodesSet.Result().ToSlice()
+	}
+	return res, nil
 }

 func (task *GuestMigrateTask) OnStartSchedule(obj IScheduleModel) {
--- a/pkg/hostman/guestman/pod.go
+++ b/pkg/hostman/guestman/pod.go
@@ -1451,15 +1451,22 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 	}

 	var cpuSetCpus string
+	var cpuSetMems string
 	{
 		cpuSets := sets.NewString()
+		cpuMemSets := sets.NewString()
 		if len(s.Desc.CpuNumaPin) > 0 {
-			for _, cpuNum := range s.GetDesc().CpuNumaPin {
-				for _, cpuPin := range cpuNum.VcpuPin {
+			for _, cpuNumaPin := range s.GetDesc().CpuNumaPin {
+				for _, cpuPin := range cpuNumaPin.VcpuPin {
 					cpuSets.Insert(fmt.Sprintf("%d", cpuPin.Pcpu))
 				}
+				if cpuNumaPin.NodeId != nil && cpuNumaPin.SizeMB > 0 {
+					cpuMemSets.Insert(fmt.Sprintf("%d", int(*cpuNumaPin.NodeId)))
+				}
 			}
+
 			cpuSetCpus = strings.Join(cpuSets.List(), ",")
+			cpuSetMems = strings.Join(cpuMemSets.List(), ",")
 		} else if len(s.Desc.VcpuPin) > 0 {
 			for _, vcpuPin := range s.Desc.VcpuPin {
 				cpuSets.Insert(vcpuPin.Pcpus)
@@ -1494,7 +1501,7 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 				MemoryLimitInBytes:     s.GetDesc().Mem * 1024 * 1024,
 				OomScoreAdj:            0,
 				CpusetCpus:             cpuSetCpus,
-				CpusetMems:             "",
+				CpusetMems:             cpuSetMems,
 				HugepageLimits:         nil,
 				Unified:                nil,
 				MemorySwapLimitInBytes: 0,
--- a/pkg/hostman/hosthandler/handler.go
+++ b/pkg/hostman/hosthandler/handler.go
@@ -103,5 +103,6 @@ func hostRestart(ctx context.Context, hostId string, body jsonutils.JSONObject)
 }

 func hostProbeIsolatedDevices(ctx context.Context, hostId string, body jsonutils.JSONObject) (interface{}, error) {
-	return hostinfo.Instance().ProbeSyncIsolatedDevices(hostId, body)
+	_, err := hostinfo.Instance().ProbeSyncIsolatedDevices(hostId, body)
+	return nil, err
 }
--- a/pkg/hostman/hostinfo/hostinfo.go
+++ b/pkg/hostman/hostinfo/hostinfo.go
@@ -2228,6 +2228,8 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		return nil, errors.Wrap(err, "getRemoteIsolatedDevices")
 	}

+	// devs need update
+	var devsNeedUpdate = map[string]bool{}
 	for _, obj := range objs {
 		info := isolated_device.CloudDeviceInfo{}
 		if err := obj.Unmarshal(&info); err != nil {
@@ -2236,10 +2238,12 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 		dev := h.IsolatedDeviceMan.GetDeviceByIdent(info.VendorDeviceId, info.Addr, info.MdevId)
 		if dev != nil {
 			dev.SetDeviceInfo(info)
+			devsNeedUpdate[dev.GetCloudId()] = h.IsolatedDeviceMan.CheckDevIsNeedUpdate(dev, &info)
 		} else {
 			// detach device
 			h.IsolatedDeviceMan.AppendDetachedDevice(&info)
 		}
+
 	}

 	h.IsolatedDeviceMan.StartDetachTask()
@@ -2247,20 +2251,28 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {

 	// sync each isolated device found
 	eg := errgroup.Group{}
+	// limits the number of active goroutines in this group to at most
+	eg.SetLimit(16)
 	mtx := sync.Mutex{}
 	updateDevs := jsonutils.NewArray()
-
 	devs := h.IsolatedDeviceMan.GetDevices()
 	for i := range devs {
 		dev := devs[i]
 		eg.Go(func() error {
-			if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil {
+			needUpdate := false
+			if need, ok := devsNeedUpdate[dev.GetCloudId()]; !ok || need {
+				needUpdate = true
+			}
+
+			if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev, needUpdate); err != nil {
 				log.Errorf("Sync deviceInfo %s error: %v", dev.String(), err)
 				return errors.Wrapf(err, "Sync device %s", dev.String())
 			} else {
-				mtx.Lock()
-				updateDevs.Add(obj)
-				mtx.Unlock()
+				if obj != nil {
+					mtx.Lock()
+					updateDevs.Add(obj)
+					mtx.Unlock()
+				}
 				return nil
 			}
 		})
--- a/pkg/hostman/isolated_device/container_device/base_dev.go
+++ b/pkg/hostman/isolated_device/container_device/base_dev.go
@@ -20,9 +20,11 @@ import (
 	"path"
 	"strings"

+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"

 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 )

 type BaseDevice struct {
@@ -70,7 +72,17 @@ func (c BaseDevice) GetNvidiaMpsThreadPercentage() int {
 }

 func (c BaseDevice) GetNumaNode() (int, error) {
-	return -1, nil
+	if c.SBaseDevice == nil {
+		return -1, nil
+	}
+
+	numaNodePath := fmt.Sprintf("/sys/bus/pci/devices/0000:%s/numa_node", c.SBaseDevice.GetOriginAddr())
+	numaNode, err := fileutils2.FileGetIntContent(numaNodePath)
+	if err != nil {
+		log.Errorf("failed get numa node %s: %s", c.SBaseDevice.GetOriginAddr(), err)
+		return -1, nil
+	}
+	return numaNode, nil
 }

 func CheckVirtualNumber(dev *isolated_device.ContainerDevice) error {
@@ -120,8 +132,11 @@ func newPCIGPURenderBaseDevice(devPath string, index int, devType isolated_devic
 				return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr)
 			}
 			dev := isolated_device.NewPCIDevice2(pciOutput[0])
-			dev.Addr = fmt.Sprintf("%s-%d", dev.Addr, index)
-			return NewBaseDevice(dev, devType, devPath), nil
+			devAddr := dev.Addr
+			baseDev := NewBaseDevice(dev, devType, devPath)
+			baseDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, index), devAddr)
+
+			return baseDev, nil
 		}
 	}
 	return nil, errors.Wrapf(errors.ErrNotFound, "%s doesn't exist in %s", devPath, dir)
--- a/pkg/hostman/isolated_device/container_device/nvidia_mps.go
+++ b/pkg/hostman/isolated_device/container_device/nvidia_mps.go
@@ -176,7 +176,8 @@ func getNvidiaMPSGpus() ([]isolated_device.IDevice, error) {
 				ThreadPercentage: 100 / options.HostOptions.CudaMPSReplicas,
 			}
 			gpuDev.SetModelName(gpuName)
-			gpuDev.SetAddr(fmt.Sprintf("%s-%d", gpuDev.GetAddr(), i))
+			devAddr := gpuDev.GetAddr()
+			gpuDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, i), devAddr)
 			devs = append(devs, gpuDev)
 		}
 	}
--- a/pkg/hostman/isolated_device/isolated_device.go
+++ b/pkg/hostman/isolated_device/isolated_device.go
@@ -38,14 +38,39 @@ const (
 )

 type CloudDeviceInfo struct {
-	Id             string `json:"id"`
-	GuestId        string `json:"guest_id"`
-	HostId         string `json:"host_id"`
-	DevType        string `json:"dev_type"`
-	VendorDeviceId string `json:"vendor_device_id"`
-	Addr           string `json:"addr"`
-	DetectedOnHost bool   `json:"detected_on_host"`
-	MdevId         string `json:"mdev_id"`
+	Id                  string                      `json:"id"`
+	GuestId             string                      `json:"guest_id"`
+	HostId              string                      `json:"host_id"`
+	DevType             string                      `json:"dev_type"`
+	VendorDeviceId      string                      `json:"vendor_device_id"`
+	Addr                string                      `json:"addr"`
+	DetectedOnHost      bool                        `json:"detected_on_host"`
+	MdevId              string                      `json:"mdev_id"`
+	Model               string                      `json:"model"`
+	WireId              string                      `json:"wire_id"`
+	OvsOffloadInterface string                      `json:"ovs_offload_interface"`
+	IsInfinibandNic     bool                        `json:"is_infiniband_nic"`
+	NvmeSizeMB          int                         `json:"nvme_size_mb"`
+	DevicePath          string                      `json:"device_path"`
+	MpsMemoryLimit      int                         `json:"mps_memory_limit"`
+	MpsMemoryTotal      int                         `json:"mps_memory_total"`
+	MpsThreadPercentage int                         `json:"mps_thread_percentage"`
+	NumaNode            int                         `json:"numa_node"`
+	PcieInfo            *api.IsolatedDevicePCIEInfo `json:"pcie_info"`
+
+	// The frame rate limiter (FRL) configuration in frames per second
+	FRL string `json:"frl"`
+	// The frame buffer size in Mbytes
+	Framebuffer string `json:"framebuffer"`
+	// The maximum resolution per display head, eg: 5120x2880
+	MaxResolution string `json:"max_resolution"`
+	// The maximum number of virtual display heads that the vGPU type supports
+	// In computer graphics and display technology, the term "head" is commonly used to
+	// describe the physical interface of a display device or display output.
+	// It refers to a connection point on the monitor, such as HDMI, DisplayPort, or VGA interface.
+	NumHeads string `json:"num_heads"`
+	// The maximum number of vGPU instances per physical GPU
+	MaxInstance string `json:"max_instance"`
 }

 type IHost interface {
@@ -127,6 +152,7 @@ type IsolatedDeviceManager interface {
 	BatchCustomProbe()
 	AppendDetachedDevice(dev *CloudDeviceInfo)
 	GetQemuParams(devAddrs []string) *QemuParams
+	CheckDevIsNeedUpdate(dev IDevice, devInfo *CloudDeviceInfo) bool
 }

 type isolatedDeviceManager struct {
@@ -454,6 +480,54 @@ func (man *isolatedDeviceManager) getSession() *mcclient.ClientSession {
 	return man.host.GetSession()
 }

+func (man *isolatedDeviceManager) CheckDevIsNeedUpdate(dev IDevice, devInfo *CloudDeviceInfo) bool {
+	if dev.GetDeviceType() != devInfo.DevType {
+		return true
+	}
+	if dev.GetModelName() != devInfo.Model {
+		return true
+	}
+	if dev.GetWireId() != devInfo.WireId {
+		return true
+	}
+	if dev.IsInfinibandNic() != devInfo.IsInfinibandNic {
+		return true
+	}
+	if dev.GetOvsOffloadInterfaceName() != devInfo.OvsOffloadInterface {
+		return true
+	}
+	if dev.GetNVMESizeMB() > 0 && devInfo.NvmeSizeMB > 0 && dev.GetNVMESizeMB() != devInfo.NvmeSizeMB {
+		return true
+	}
+	if numaNode, _ := dev.GetNumaNode(); numaNode != devInfo.NumaNode {
+		return true
+	}
+	if dev.GetMdevId() != devInfo.MdevId {
+		return true
+	}
+	if info := dev.GetPCIEInfo(); info != nil && devInfo.PcieInfo == nil {
+		return true
+	}
+	if profile := dev.GetNVIDIAVgpuProfile(); profile != nil {
+		if val, _ := profile["frl"]; val != devInfo.FRL {
+			return true
+		}
+		if val, _ := profile["framebuffer"]; val != devInfo.Framebuffer {
+			return true
+		}
+		if val, _ := profile["max_resolution"]; val != devInfo.MaxResolution {
+			return true
+		}
+		if val, _ := profile["num_heads"]; val != devInfo.NumHeads {
+			return true
+		}
+		if val, _ := profile["max_instance"]; val != devInfo.MaxInstance {
+			return true
+		}
+	}
+	return false
+}
+
 func (man *isolatedDeviceManager) GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice {
 	for _, dev := range man.devices {
 		if dev.GetVendorDeviceId() == vendorDevId && dev.GetAddr() == addr && dev.GetMdevId() == mdevId {
@@ -528,6 +602,7 @@ func (man *isolatedDeviceManager) GetQemuParams(devAddrs []string) *QemuParams {

 type SBaseDevice struct {
 	dev            *PCIDevice
+	originAddr     string
 	cloudId        string
 	hostId         string
 	guestId        string
@@ -577,12 +652,17 @@ func (dev *SBaseDevice) SetDeviceInfo(info CloudDeviceInfo) {
 	}
 }

-func SyncDeviceInfo(session *mcclient.ClientSession, hostId string, dev IDevice) (jsonutils.JSONObject, error) {
+func SyncDeviceInfo(session *mcclient.ClientSession, hostId string, dev IDevice, needUpdate bool) (jsonutils.JSONObject, error) {
 	if len(dev.GetHostId()) == 0 {
 		dev.SetHostId(hostId)
 	}
 	data := GetApiResourceData(dev)
 	if len(dev.GetCloudId()) != 0 {
+		if !needUpdate {
+			log.Infof("Update %s isolated_device: do nothing", dev.GetCloudId())
+			return nil, nil
+		}
+
 		log.Infof("Update %s isolated_device: %s", dev.GetCloudId(), data.String())
 		return modules.IsolatedDevices.Update(session, dev.GetCloudId(), data)
 	}
@@ -602,7 +682,15 @@ func (dev *SBaseDevice) GetAddr() string {
 	return dev.dev.Addr
 }

-func (dev *SBaseDevice) SetAddr(addr string) {
+func (dev *SBaseDevice) GetOriginAddr() string {
+	if dev.originAddr != "" {
+		return dev.originAddr
+	}
+	return dev.dev.Addr
+}
+
+func (dev *SBaseDevice) SetAddr(addr, originAddr string) {
+	dev.originAddr = originAddr
 	dev.dev.Addr = addr
 }

--- a/pkg/scheduler/cache/candidate/baremetals.go
+++ b/pkg/scheduler/cache/candidate/baremetals.go
@@ -115,7 +115,7 @@ func (bd *BaremetalDesc) IndexKey() string {
 	return bd.Id
 }

-func (bd *BaremetalDesc) AllocCpuNumaPin(vcpuCount, memSizeKB int) []scheduler.SCpuNumaPin {
+func (bd *BaremetalDesc) AllocCpuNumaPin(vcpuCount, memSizeKB int, preferNumaNodes []int) []scheduler.SCpuNumaPin {
 	return nil
 }

--- a/pkg/scheduler/cache/candidate/common.go
+++ b/pkg/scheduler/cache/candidate/common.go
@@ -35,6 +35,9 @@ var (
 		computeapi.VM_BACKUP_STARTING,
 		computeapi.POD_STATUS_CONTAINER_EXITED,
 		computeapi.POD_STATUS_CRASH_LOOP_BACK_OFF,
+		computeapi.POD_STATUS_STARTING_CONTAINER,
+		computeapi.POD_STATUS_STOP_CONTAINER_FAILED,
+		computeapi.POD_STATUS_STOPPING_CONTAINER,
 	)

 	VMCreatingStatus = sets.NewString(
@@ -45,6 +48,18 @@ var (
 		computeapi.VM_DEPLOYING,
 		computeapi.VM_BACKUP_CREATING,
 		computeapi.VM_DEPLOYING_BACKUP,
+		computeapi.POD_STATUS_CREATING_CONTAINER,
+	)
+
+	VMStoppedStatus = sets.NewString(
+		computeapi.VM_READY,
+		computeapi.VM_START_FAILED,
+		computeapi.VM_SCHEDULE_FAILED,
+		computeapi.VM_NETWORK_FAILED,
+		computeapi.VM_CREATE_FAILED,
+		computeapi.VM_DISK_FAILED,
+		computeapi.POD_STATUS_START_CONTAINER_FAILED,
+		computeapi.POD_STATUS_CREATE_CONTAINER_FAILED,
 	)
 )

@@ -78,6 +93,10 @@ func IsGuestPendingDelete(g models.SGuest) bool {
 	return g.PendingDeleted
 }

+func IsGuestStoppedStatus(g models.SGuest) bool {
+	return VMStoppedStatus.Has(g.Status)
+}
+
 func ToDict[O lockman.ILockedObject](objs []O) map[string]*O {
 	ret := make(map[string]*O, 0)
 	for _, obj := range objs {
--- a/pkg/scheduler/cache/candidate/hosts.go
+++ b/pkg/scheduler/cache/candidate/hosts.go
@@ -72,7 +72,7 @@ func (h *hostGetter) FreeMemorySize(useRsvd bool) int64 {
 }

 func (h *hostGetter) NumaAllocateEnabled() bool {
-	return h.h.HostTopo.NumaEnabled
+	return h.h.HostTopo.NumaEnabled || h.h.HostType == computeapi.HOST_TYPE_CONTAINER
 }

 func (h *hostGetter) GetFreeCpuNuma() []*scheduler.SFreeNumaCpuMem {
@@ -268,11 +268,79 @@ type NumaNode struct {
 	CpuCount          int

 	NodeId                int
-	NumaHugeMemSizeKB     int
-	NumaHugeFreeMemSizeKB int
+	Distances             []int
+	NumaNodeMemSizeKB     int
+	NumaNodeFreeMemSizeKB int
+}
+
+func (n *NumaNode) nodeEnough(vcpuCount, memSizeKB, cmtBound int, enableNumaAlloc bool) bool {
+	if n.CpuCount*cmtBound-n.VcpuCount < vcpuCount {
+		return false
+	}
+	if enableNumaAlloc {
+		if n.NumaNodeFreeMemSizeKB < memSizeKB {
+			return false
+		}
+	}
+	return true
+}
+
+func (n *NumaNode) allocCpusetSequenceN(vcpuCount int, usedCpu map[int]int) {
+	var seqNumber = o.Options.GuestCpusetAllocSequenceInterval
+	if vcpuCount%seqNumber != 0 || n.CpuCount/len(n.CpuDies) < vcpuCount {
+		n._allocCpuset(vcpuCount, usedCpu)
+		return
+	}
+
+	for i := range n.CpuDies {
+		detectedSet := cpuset.NewCPUSet()
+		for j := range n.CpuDies[i].CpuFree {
+			if detectedSet.Contains(n.CpuDies[i].CpuFree[j].Cpu) {
+				continue
+			}
+
+			cpuIdBase := n.CpuDies[i].CpuFree[j].Cpu - n.CpuDies[i].CpuFree[j].Cpu%vcpuCount
+			lo, hi := cpuIdBase, cpuIdBase+vcpuCount-1
+			cpuIds := make([]int, hi-lo+1)
+			for m := range cpuIds {
+				cpuIds[m] = m + lo
+			}
+
+			var matched = true
+			cpuIdSet := cpuset.NewCPUSet(cpuIds...)
+			detectedSet = detectedSet.Union(cpuIdSet)
+			for k := range n.CpuDies[i].CpuFree {
+				if !cpuIdSet.Contains(n.CpuDies[i].CpuFree[k].Cpu) {
+					continue
+				}
+				if n.CpuDies[i].CpuFree[k].Free <= 0 {
+					matched = false
+					break
+				}
+			}
+			if !matched {
+				continue
+			}
+
+			for m := range cpuIds {
+				usedCpu[cpuIds[m]] = 1
+			}
+			return
+		}
+	}
+
+	n._allocCpuset(vcpuCount, usedCpu)
 }

 func (n *NumaNode) allocCpuset(vcpuCount int, usedCpu map[int]int) {
+	if o.Options.GuestCpusetAllocSequence {
+		n.allocCpusetSequenceN(vcpuCount, usedCpu)
+		return
+	}
+	n._allocCpuset(vcpuCount, usedCpu)
+}
+
+func (n *NumaNode) _allocCpuset(vcpuCount int, usedCpu map[int]int) {
 	for i := range n.CpuDies {
 		for j := range n.CpuDies[i].CpuFree {
 			cpuId, nFree := n.CpuDies[i].CpuFree[j].Cpu, n.CpuDies[i].CpuFree[j].Free
@@ -320,7 +388,7 @@ func (n *NumaNode) allocCpuset(vcpuCount int, usedCpu map[int]int) {
 		}
 		//sort.Sort(n.CpuDies[i].CpuFree)
 	}
-	n.allocCpuset(vcpuCount, usedCpu)
+	n._allocCpuset(vcpuCount, usedCpu)
 }

 func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
@@ -341,18 +409,23 @@ func (n *NumaNode) AllocCpuset(vcpuCount int) []int {
 	return ret
 }

-func NewNumaNode(nodeId, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr) *NumaNode {
+func NewNumaNode(nodeId int, nodeDistances []int, hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr, memSizeKB, memCmtBound int) *NumaNode {
 	n := new(NumaNode)
 	n.LogicalProcessors = cpuset.NewCPUSet()
 	n.NodeId = nodeId
+	n.Distances = nodeDistances

-	for i := range nodeHugepages {
-		if nodeHugepages[i].NodeId == nodeId {
-			n.NumaHugeMemSizeKB = nodeHugepages[i].HugepageNr * hugepageSizeKb
+	if len(nodeHugepages) > 0 {
+		for i := range nodeHugepages {
+			if nodeHugepages[i].NodeId == nodeId {
+				n.NumaNodeMemSizeKB = nodeHugepages[i].HugepageNr * hugepageSizeKb
+			}
 		}
+	} else {
+		n.NumaNodeMemSizeKB = memSizeKB * memCmtBound
 	}

-	n.NumaHugeFreeMemSizeKB = n.NumaHugeMemSizeKB
+	n.NumaNodeFreeMemSizeKB = n.NumaNodeMemSizeKB
 	return n
 }

@@ -360,6 +433,7 @@ type SHostTopo struct {
 	Nodes       []*NumaNode
 	NumaEnabled bool
 	CPUCmtbound int
+	HostName    string
 }

 func HostTopoSubPendingUsage(topo *SHostTopo, cpuUsage map[int]int, numaMemUsage map[int]int) *SHostTopo {
@@ -373,11 +447,12 @@ func HostTopoSubPendingUsage(topo *SHostTopo, cpuUsage map[int]int, numaMemUsage
 		res.Nodes[i].VcpuCount = topo.Nodes[i].VcpuCount
 		res.Nodes[i].CpuCount = topo.Nodes[i].CpuCount
 		res.Nodes[i].NodeId = topo.Nodes[i].NodeId
-		res.Nodes[i].NumaHugeMemSizeKB = topo.Nodes[i].NumaHugeMemSizeKB
-		res.Nodes[i].NumaHugeFreeMemSizeKB = topo.Nodes[i].NumaHugeFreeMemSizeKB
+		res.Nodes[i].NumaNodeMemSizeKB = topo.Nodes[i].NumaNodeMemSizeKB
+		res.Nodes[i].NumaNodeFreeMemSizeKB = topo.Nodes[i].NumaNodeFreeMemSizeKB
+		res.Nodes[i].Distances = topo.Nodes[i].Distances

 		if memUsed, ok := numaMemUsage[topo.Nodes[i].NodeId]; ok {
-			res.Nodes[i].NumaHugeMemSizeKB -= memUsed * 1024
+			res.Nodes[i].NumaNodeFreeMemSizeKB -= memUsed * 1024
 		}
 		res.Nodes[i].CpuDies = make([]*CPUDie, len(topo.Nodes[i].CpuDies))
 		for j := range topo.Nodes[i].CpuDies {
@@ -412,12 +487,12 @@ func (pq SHostTopo) Len() int { return len(pq.Nodes) }

 func (pq SHostTopo) Less(i, j int) bool {
 	if pq.NumaEnabled {
-		if pq.Nodes[i].NumaHugeFreeMemSizeKB == pq.Nodes[j].NumaHugeFreeMemSizeKB {
+		if pq.Nodes[i].NumaNodeFreeMemSizeKB == pq.Nodes[j].NumaNodeFreeMemSizeKB {
 			return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
 		}
-		return pq.Nodes[i].NumaHugeFreeMemSizeKB > pq.Nodes[j].NumaHugeFreeMemSizeKB
+		return pq.Nodes[i].NumaNodeFreeMemSizeKB > pq.Nodes[j].NumaNodeFreeMemSizeKB
 	} else {
-		return pq.Nodes[i].VcpuCount < pq.Nodes[j].VcpuCount
+		return pq.Nodes[i].NumaNodeFreeMemSizeKB > pq.Nodes[j].NumaNodeFreeMemSizeKB
 	}
 }

@@ -440,8 +515,8 @@ func (h *SHostTopo) LoadCpuNumaPin(guestsCpuNumaPin []scheduler.SCpuNumaPin) {

 		cpus := gCpuNumaPin.CpuPin
 		node.CpuDies.LoadCpus(cpus, len(cpus))
-		if h.NumaEnabled && gCpuNumaPin.MemSizeMB != nil {
-			node.NumaHugeFreeMemSizeKB -= *gCpuNumaPin.MemSizeMB * 1024
+		if gCpuNumaPin.MemSizeMB != nil {
+			node.NumaNodeFreeMemSizeKB -= *gCpuNumaPin.MemSizeMB * 1024
 		}
 		node.VcpuCount += len(cpus)
 	}
@@ -455,7 +530,7 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool {

 	for i := 0; i < nodeCount; i++ {
 		if h.NumaEnabled {
-			if h.Nodes[i].NumaHugeFreeMemSizeKB < leastFree {
+			if h.Nodes[i].NumaNodeFreeMemSizeKB < leastFree {
 				return false
 			}
 		}
@@ -473,18 +548,96 @@ func (h *SHostTopo) nodesEnough(nodeCount, vcpuCount int, memSizeKB int) bool {
 	return true
 }

-func (h *SHostTopo) AllocCpuNumaNodes(vcpuCount, memSizeKB int) []scheduler.SCpuNumaPin {
+func (h *SHostTopo) allocCpuNumaNodesByPreferNodes(
+	vcpuCount, memSizeKB, nodeCount int, preferNumaNodes []int, sortedNumaDistance []SSortedNumaDistance,
+) []scheduler.SCpuNumaPin {
+	res := make([]scheduler.SCpuNumaPin, 0)
+	var nodeAllocSize = memSizeKB / nodeCount
+	var pcpuCount = vcpuCount / nodeCount
+	var remPcpuCount = vcpuCount % nodeCount
+
+	allocatedNode := 0
+	for i := range sortedNumaDistance {
+		if allocatedNode >= nodeCount {
+			break
+		}
+
+		var npcpuCount = pcpuCount
+		if remPcpuCount > 0 {
+			npcpuCount += 1
+			remPcpuCount -= 1
+		}
+		nodeIdx := sortedNumaDistance[i].NodeIndex
+		if h.Nodes[nodeIdx].nodeEnough(vcpuCount, memSizeKB, h.CPUCmtbound, h.NumaEnabled) {
+			cpuNumaPin := scheduler.SCpuNumaPin{
+				CpuPin: h.Nodes[nodeIdx].AllocCpuset(npcpuCount),
+				NodeId: h.Nodes[nodeIdx].NodeId,
+			}
+			allocSize := nodeAllocSize / 1024
+			cpuNumaPin.MemSizeMB = &allocSize
+			res = append(res, cpuNumaPin)
+			allocatedNode += 1
+		} else {
+			log.Infof("%s node %v not enough", h.HostName, h.Nodes[i])
+		}
+
+		log.Infof("node %d, free mems %d", h.Nodes[nodeIdx].NodeId, h.Nodes[nodeIdx].NumaNodeFreeMemSizeKB)
+	}
+
+	if allocatedNode < nodeCount {
+		return nil
+	}
+	return res
+}
+
+type SSortedNumaDistance struct {
+	NodeIndex int
+	Distance  int
+}
+
+func (h *SHostTopo) getDistancesSeqByPreferNodes(preferNumaNodes []int) []SSortedNumaDistance {
+	sortedNumaDistance := make([]SSortedNumaDistance, len(h.Nodes))
+	for i := range h.Nodes {
+		distance := 0
+		for j := range preferNumaNodes {
+			log.Infof("node distance %v", h.Nodes[i].Distances)
+			distance += h.Nodes[i].Distances[preferNumaNodes[j]]
+		}
+		sortedNumaDistance[i] = SSortedNumaDistance{
+			NodeIndex: i,
+			Distance:  distance,
+		}
+	}
+	sort.Slice(sortedNumaDistance, func(i, j int) bool {
+		return sortedNumaDistance[i].Distance < sortedNumaDistance[j].Distance
+	})
+	return sortedNumaDistance
+}
+
+func (h *SHostTopo) AllocCpuNumaNodes(vcpuCount, memSizeKB int, ignoreMemSingular bool, preferNumaNodes []int) []scheduler.SCpuNumaPin {
+	if h.NumaEnabled && len(preferNumaNodes) > 0 {
+		log.Infof("preferNumaNodes %v", preferNumaNodes)
+		sortedNumaDistance := h.getDistancesSeqByPreferNodes(preferNumaNodes)
+		for nodeCount := 1; nodeCount <= len(h.Nodes); nodeCount *= 2 {
+			ret := h.allocCpuNumaNodesByPreferNodes(vcpuCount, memSizeKB, nodeCount, preferNumaNodes, sortedNumaDistance)
+			if ret != nil {
+				return ret
+			}
+		}
+	}
+
 	res := make([]scheduler.SCpuNumaPin, 0)
 	for nodeCount := 1; nodeCount <= len(h.Nodes); nodeCount *= 2 {
 		if ok := h.nodesEnough(nodeCount, vcpuCount, memSizeKB); !ok {
-			log.Infof("node count %d not enough", nodeCount)
+			log.Infof("host %s node count %d not enough", h.HostName, nodeCount)
 			continue
 		}
 		log.Infof("use node count %d", nodeCount)

 		var nodeAllocSize = memSizeKB / nodeCount
-		if h.NumaEnabled {
+		if h.NumaEnabled && !ignoreMemSingular {
 			if nodeAllocSize/1024%1024 > 0 {
+				log.Infof("host %s node alloc size singular %d", h.HostName, nodeAllocSize)
 				continue
 			}
 		}
@@ -543,12 +696,22 @@ func (b *HostBuilder) buildHostTopo(
 	hugepageSizeKb int, nodeHugepages []hostapi.HostNodeHugepageNr,
 	info *hostapi.HostTopology,
 ) error {
+	var numaEnabled = len(nodeHugepages) > 0
 	hostTopo := new(SHostTopo)
 	hostTopo.Nodes = make([]*NumaNode, len(info.Nodes))

 	hasL3Cache := false
 	for i := 0; i < len(info.Nodes); i++ {
-		node := NewNumaNode(info.Nodes[i].ID, hugepageSizeKb, nodeHugepages)
+		nodoMemSizeKB := 0
+		if info.Nodes[i].Memory != nil {
+			nodoMemSizeKB = int(info.Nodes[i].Memory.TotalUsableBytes/1024) - (desc.MemReserved * 1024 / len(info.Nodes))
+			if desc.HostType == computeapi.HOST_TYPE_CONTAINER && o.Options.ContainerNumaAllocate {
+				numaEnabled = true
+				log.Infof("host %s ignore singular", desc.Name)
+			}
+		}
+
+		node := NewNumaNode(info.Nodes[i].ID, info.Nodes[i].Distances, hugepageSizeKb, nodeHugepages, nodoMemSizeKB, int(desc.MemCmtbound))

 		cpuDies := make([]*CPUDie, 0)
 		for j := 0; j < len(info.Nodes[i].Caches); j++ {
@@ -606,9 +769,8 @@ func (b *HostBuilder) buildHostTopo(
 		hostTopo.Nodes[i] = node
 	}
 	hostTopo.CPUCmtbound = int(desc.CPUCmtbound)
-	if len(nodeHugepages) > 0 {
-		hostTopo.NumaEnabled = true
-	}
+	hostTopo.NumaEnabled = numaEnabled
+	hostTopo.HostName = desc.Name

 	desc.HostTopo = hostTopo
 	//log.Infof("host topo %s", jsonutils.Marshal(hostTopo))
@@ -838,7 +1000,7 @@ func (h *HostDesc) GetFreeCpuNuma() scheduler.SortedFreeNumaCpuMam {
 		nodeFree := new(scheduler.SFreeNumaCpuMem)
 		nodeFree.NodeId = h.HostTopo.Nodes[i].NodeId
 		nodeFree.CpuCount = h.HostTopo.Nodes[i].CpuCount
-		nodeFree.MemSize = h.HostTopo.Nodes[i].NumaHugeFreeMemSizeKB / 1024
+		nodeFree.MemSize = h.HostTopo.Nodes[i].NumaNodeFreeMemSizeKB / 1024
 		nodeFree.EnableNumaAllocate = h.HostTopo.NumaEnabled
 		nodeFree.FreeCpuCount = h.HostTopo.Nodes[i].CpuCount*int(h.CPUCmtbound) - h.HostTopo.Nodes[i].VcpuCount
 		for cpuId, pending := range cpuPin {
@@ -892,7 +1054,7 @@ func (h *HostDesc) IndexKey() string {
 	return h.Id
 }

-func (h *HostDesc) AllocCpuNumaPin(vcpuCount, memSizeKB int) []scheduler.SCpuNumaPin {
+func (h *HostDesc) AllocCpuNumaPin(vcpuCount, memSizeKB int, preferNumaNodes []int) []scheduler.SCpuNumaPin {
 	if !h.EnableCpuNumaAllocate {
 		return nil
 	}
@@ -902,8 +1064,8 @@ func (h *HostDesc) AllocCpuNumaPin(vcpuCount, memSizeKB int) []scheduler.SCpuNum
 	if len(pendingUsage.CpuPin) > 0 || len(pendingUsage.NumaMemPin) > 0 {
 		hostTopo = HostTopoSubPendingUsage(h.HostTopo, pendingUsage.CpuPin, pendingUsage.NumaMemPin)
 	}
-
-	return hostTopo.AllocCpuNumaNodes(vcpuCount, memSizeKB)
+	ignoreMemSingular := h.HostType == computeapi.HOST_TYPE_CONTAINER && o.Options.ContainerNumaAllocate
+	return hostTopo.AllocCpuNumaNodes(vcpuCount, memSizeKB, ignoreMemSingular, preferNumaNodes)
 }

 func (h *HostDesc) AllocCpuNumaPinWithNodeCount(vcpuCount, memSizeKB, nodeCount int) []scheduler.SCpuNumaPin {
@@ -1352,7 +1514,22 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels
 		} else {
 			desc.Tenants[projectId] = 1
 		}
-		if IsGuestRunning(guest) {
+		if IsGuestPendingDelete(guest) {
+			memFakeDeletedSize += int64(guest.VmemSize)
+			cpuFakeDeletedCount += int64(guest.VcpuCount)
+		} else if IsGuestCreating(guest) {
+			creatingGuestCount++
+			creatingMemSize += int64(guest.VmemSize)
+			creatingCPUCount += int64(guest.VcpuCount)
+			if host.EnableNumaAllocate && guest.CpuNumaPin != nil {
+				cpuNumaPin := make([]scheduler.SCpuNumaPin, 0)
+				if err := guest.CpuNumaPin.Unmarshal(&cpuNumaPin); err != nil {
+					return errors.Wrap(err, "unmarshal cpu numa pin")
+				}
+				guestsCpuNumaPin = append(guestsCpuNumaPin, cpuNumaPin...)
+			}
+		} else if !IsGuestStoppedStatus(guest) {
+			// running status
 			runningCount++
 			memSize += int64(guest.VmemSize)
 			cpuCount += int64(guest.VcpuCount)
@@ -1363,14 +1540,34 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels
 				}
 				guestsCpuNumaPin = append(guestsCpuNumaPin, cpuNumaPin...)
 			}
-		} else if IsGuestCreating(guest) {
-			creatingGuestCount++
-			creatingMemSize += int64(guest.VmemSize)
-			creatingCPUCount += int64(guest.VcpuCount)
-		} else if IsGuestPendingDelete(guest) {
-			memFakeDeletedSize += int64(guest.VmemSize)
-			cpuFakeDeletedCount += int64(guest.VcpuCount)
 		}
+
+		//if IsGuestRunning(guest) {
+		//	runningCount++
+		//	memSize += int64(guest.VmemSize)
+		//	cpuCount += int64(guest.VcpuCount)
+		//	if host.EnableNumaAllocate && guest.CpuNumaPin != nil {
+		//		cpuNumaPin := make([]scheduler.SCpuNumaPin, 0)
+		//		if err := guest.CpuNumaPin.Unmarshal(&cpuNumaPin); err != nil {
+		//			return errors.Wrap(err, "unmarshal cpu numa pin")
+		//		}
+		//		guestsCpuNumaPin = append(guestsCpuNumaPin, cpuNumaPin...)
+		//	}
+		//} else if IsGuestCreating(guest) {
+		//	creatingGuestCount++
+		//	creatingMemSize += int64(guest.VmemSize)
+		//	creatingCPUCount += int64(guest.VcpuCount)
+		//	if host.EnableNumaAllocate && guest.CpuNumaPin != nil {
+		//		cpuNumaPin := make([]scheduler.SCpuNumaPin, 0)
+		//		if err := guest.CpuNumaPin.Unmarshal(&cpuNumaPin); err != nil {
+		//			return errors.Wrap(err, "unmarshal cpu numa pin")
+		//		}
+		//		guestsCpuNumaPin = append(guestsCpuNumaPin, cpuNumaPin...)
+		//	}
+		//} else if IsGuestPendingDelete(guest) {
+		//	memFakeDeletedSize += int64(guest.VmemSize)
+		//	cpuFakeDeletedCount += int64(guest.VcpuCount)
+		//}
 		guestCount++
 		cpuReqCount += int64(guest.VcpuCount)
 		memReqSize += int64(guest.VmemSize)
@@ -1388,7 +1585,7 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels
 	if host.EnableNumaAllocate && len(guestsCpuNumaPin) > 0 {
 		desc.HostTopo.LoadCpuNumaPin(guestsCpuNumaPin)
 	}
-	log.Infof("host topo %s", jsonutils.Marshal(desc.HostTopo))
+	//log.Infof("host %s topo %s", jsonutils.Marshal(desc.HostTopo))

 	desc.GuestCount = guestCount
 	desc.CreatingGuestCount = creatingGuestCount
--- a/pkg/scheduler/core/result.go
+++ b/pkg/scheduler/core/result.go
@@ -93,7 +93,7 @@ func (item *SchedResultItem) selectCpuNumaPin() []schedapi.SCpuNumaPin {
 	if item.SchedData.LiveMigrate && len(item.SchedData.CpuNumaPin) > 0 {
 		return item.Candidater.AllocCpuNumaPinWithNodeCount(item.SchedData.Ncpu, item.SchedData.Memory, len(item.SchedData.CpuNumaPin))
 	}
-	return item.Candidater.AllocCpuNumaPin(item.SchedData.Ncpu, item.SchedData.Memory*1024)
+	return item.Candidater.AllocCpuNumaPin(item.SchedData.Ncpu, item.SchedData.Memory*1024, item.SchedData.PreferNumaNodes)
 }

 func (item *SchedResultItem) getDisks(used *StorageUsed) []*schedapi.CandidateDisk {
--- a/pkg/scheduler/core/types.go
+++ b/pkg/scheduler/core/types.go
@@ -142,7 +142,7 @@ type Candidater interface {
 	GetSchedDesc() *jsonutils.JSONDict
 	GetGuestCount() int64
 	GetResourceType() string
-	AllocCpuNumaPin(vcpuCount, memSizeKB int) []schedapi.SCpuNumaPin
+	AllocCpuNumaPin(vcpuCount, memSizeKB int, preferNumaNodes []int) []schedapi.SCpuNumaPin
 	AllocCpuNumaPinWithNodeCount(vcpuCount, memSizeKB, nodeCount int) []schedapi.SCpuNumaPin
 }

--- a/pkg/scheduler/options/options.go
+++ b/pkg/scheduler/options/options.go
@@ -90,6 +90,10 @@ type SchedOptions struct {

 	SkuRefreshInterval string `help:"Server SKU refresh interval" default:"12h"`

+	ContainerNumaAllocate            bool `help:"Allocate numa pin for container guests" default:"false"`
+	GuestCpusetAllocSequence         bool `help:"Guest alloc cpuset sequence" default:"false"`
+	GuestCpusetAllocSequenceInterval int  `help:"Guest alloc cpuset sequence interval" default:"4"`
+
 	OpenstackOptions
 }

--- a/pkg/scheduler/test/mock/core.go
+++ b/pkg/scheduler/test/mock/core.go
@@ -888,16 +888,16 @@ func (mr *MockCandidaterMockRecorder) IndexKey() *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IndexKey", reflect.TypeOf((*MockCandidater)(nil).IndexKey))
 }

-func (m *MockCandidater) AllocCpuNumaPin(arg0, arg1 int) []scheduler.SCpuNumaPin {
+func (m *MockCandidater) AllocCpuNumaPin(arg0, arg1 int, arg2 []int) []scheduler.SCpuNumaPin {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "AllocCpuNumaPin", arg0, arg1)
+	ret := m.ctrl.Call(m, "AllocCpuNumaPin", arg0, arg1, arg2)
 	ret0, _ := ret[0].([]scheduler.SCpuNumaPin)
 	return ret0
 }

-func (mr *MockCandidaterMockRecorder) AllocCpuNumaPin(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockCandidaterMockRecorder) AllocCpuNumaPin(arg0, arg1, arg3 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocCpuNumaPin", reflect.TypeOf((*MockCandidater)(nil).AllocCpuNumaPin), arg0, arg1)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocCpuNumaPin", reflect.TypeOf((*MockCandidater)(nil).AllocCpuNumaPin), arg0, arg1, arg3)
 }

 func (m *MockCandidater) AllocCpuNumaPinWithNodeCount(arg0, arg1, arg2 int) []scheduler.SCpuNumaPin {
--- a/pkg/scheduler/test/prepare.go
+++ b/pkg/scheduler/test/prepare.go
@@ -83,7 +83,7 @@ func buildCandidate(ctrl *gomock.Controller, param sGetterParams) *mock.MockCand
 	cn.EXPECT().Getter().AnyTimes().Return(getter)
 	cn.EXPECT().IndexKey().AnyTimes().Return(getter.Id())
 	cn.EXPECT().GetResourceType().AnyTimes().Return(getter.ResourceType())
-	cn.EXPECT().AllocCpuNumaPin(gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+	cn.EXPECT().AllocCpuNumaPin(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 	return cn
 }