diff --git a/cmd/climc/shell/compute/isolatedevices.go b/cmd/climc/shell/compute/isolatedevices.go index aa99779a22..5808f185c0 100644 --- a/cmd/climc/shell/compute/isolatedevices.go +++ b/cmd/climc/shell/compute/isolatedevices.go @@ -26,8 +26,12 @@ func init() { cmd.List(&compute.DeviceListOptions{}) cmd.Update(&compute.DeviceUpdateOptions{}) cmd.Show(&compute.DeviceShowOptions{}) + cmd.Create(&compute.DeviceCreateOptions{}) cmd.BatchDelete(&compute.DeviceDeleteOptions{}) cmd.BatchPerform("purge", &compute.DevicePurgeOptions{}) cmd.Perform("public", &options.SharableResourcePublicOptions{}) cmd.Perform("private", &options.BaseIdOptions{}) + + scmd := shell.NewResourceCmd(&modules.ServerIsolatedDevices) + scmd.List(&compute.ServerDeviceListOptions{}) } diff --git a/cmd/climc/shell/compute/serverisolatedevices.go b/cmd/climc/shell/compute/serverisolatedevices.go index c3ff90e137..1e0a3edbf6 100644 --- a/cmd/climc/shell/compute/serverisolatedevices.go +++ b/cmd/climc/shell/compute/serverisolatedevices.go @@ -23,15 +23,15 @@ import ( func init() { type ServerAttachDeviceOptions struct { - SERVER string `help:"ID or name of server"` - DEVICE string `help:"ID of isolated device to attach"` - Type string `help:"Device type" choices:"GPU-HPC|GPU-VGA|PCI"` + SERVER string `help:"ID or name of server"` + DEVICE string `help:"ID of isolated device to attach"` + GpuType string `help:"Type of GPU to attach to" choices:"HPC|VGA"` } R(&ServerAttachDeviceOptions{}, "server-attach-isolated-device", "Attach an existing isolated device to a virtual server", func(s *mcclient.ClientSession, args *ServerAttachDeviceOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.DEVICE), "device") - if len(args.Type) > 0 { - params.Add(jsonutils.NewString(args.Type), "dev_type") + if args.GpuType != "" { + params.Add(jsonutils.NewString(args.GpuType), "gpu_type") } srv, err := modules.Servers.PerformAction(s, args.SERVER, "attach-isolated-device", params) if err != nil { @@ -44,11 +44,13 @@ func init() { type ServerDetachDeviceOptions struct { SERVER string `help:"ID or name of server"` DEVICE string `help:"ID of isolated device to detach"` + INDEX int `help:"Index of isolated device to detach"` IsForce bool `help:"Force detach isolated device"` } R(&ServerDetachDeviceOptions{}, "server-detach-isolated-device", "Detach a isolated device from a virtual server", func(s *mcclient.ClientSession, args *ServerDetachDeviceOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.DEVICE), "device") + params.Add(jsonutils.NewInt(int64(args.INDEX)), "index") if args.IsForce { params.Set("is_force", jsonutils.JSONTrue) } diff --git a/cmd/mps-daemon/main.go b/cmd/mps-daemon/main.go index 28d364859e..f6a6d24198 100644 --- a/cmd/mps-daemon/main.go +++ b/cmd/mps-daemon/main.go @@ -31,6 +31,7 @@ import ( "yunion.io/x/pkg/util/signalutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device" "yunion.io/x/onecloud/pkg/hostman/options" @@ -67,7 +68,7 @@ func NewDaemon() (*Daemon, error) { } for i := range devCfg.Devices { if devCfg.Devices[i].Type == isolated_device.ContainerDeviceTypeNvidiaMps { - dev, err := container_device.NewPCIGPURenderBaseDevice(devCfg.Devices[i].Path, 0, isolated_device.ContainerDeviceTypeNvidiaMps) + dev, err := container_device.NewPCIGPURenderBaseDevice(devCfg.Devices[i].Path, 0, api.GPU_TYPE, api.DEVICE_SHARING_MODE_MPS) if err != nil { return nil, errors.Wrapf(err, "parse pci device %s", devCfg.Devices[i].Path) } diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index a67accbdbc..fca07e5be4 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -315,17 +315,22 @@ type IsolatedDeviceConfig struct { Id string `json:"id"` DevType string `json:"dev_type"` Model string `json:"model"` + SharingMode string `json:"sharing_mode"` Vendor string `json:"vendor"` NetworkIndex *int `json:"network_index"` WireId string `json:"wire_id"` DiskIndex *int8 `json:"disk_index"` DevicePath string `json:"device_path"` + GpuType string `json:"gpu_type"` // MemoryMb is the minimum on-device memory in MiB required from the // candidate isolated_device (e.g. NVIDIA GPU VRAM). 0 means no constraint. // The scheduler excludes devices whose memory_size > 0 and is below this // threshold; devices with memory_size == 0 are treated as unknown and // allowed through to avoid penalising hosts that haven't reported yet. MemoryMb int `json:"memory_mb,omitempty"` + // Memory request for Devices allocate by Memory size + MemoryRequest int `json:"memory_request,omitempty"` + SmUtilLimit int `json:"sm_util_limit,omitempty"` } type BaremetalDiskConfig struct { diff --git a/pkg/apis/compute/container.go b/pkg/apis/compute/container.go index eff41ce6cb..6e1c46b1d2 100644 --- a/pkg/apis/compute/container.go +++ b/pkg/apis/compute/container.go @@ -40,6 +40,7 @@ const ( CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU" CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS" CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE" + CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI" CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" ) @@ -49,14 +50,16 @@ var ( CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_VASTAITECH_GPU, } ) -var NVIDIA_GPU_TYPES = []string{ +var CONTAINER_NVIDIA_GPU_TYPES = []string{ CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_NVIDIA_GPU_SHARE, } @@ -198,10 +201,11 @@ type ContainerHostDevice struct { } type ContainerIsolatedDevice struct { - Index *int `json:"index"` - Id string `json:"id"` - OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` - CDI *apis.ContainerIsolatedDeviceCDI `json:"cdi"` + Index *int `json:"index"` + Id string `json:"id"` + GuestIsolatedDeviceIndex int `json:"guest_isolated_device_index"` + OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` + CDI *apis.ContainerIsolatedDeviceCDI `json:"cdi"` } type ContainerDevice struct { diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index b9b72e4115..9e24b5634c 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -133,6 +133,8 @@ type ServerListInput struct { SnapshotpolicyId string `json:"snapshotpolicy_id"` + IsolatedDeviceId string `json:"isolated_device_id"` + // 是否调度到宿主机上 WithHost *bool `json:"with_host"` @@ -1574,8 +1576,25 @@ func (conf ServerChangeConfigSettings) AddedDisk() int { } type ServerReleasedIsolatedDevice struct { - DevType string `json:"dev_type"` - Model string `json:"model"` + DevType string `json:"dev_type"` + Model string `json:"model"` + GpuType string `json:"gpu_type"` + SharingMode string `json:"sharing_mode"` + MemoryRequest int `json:"memory_request"` +} + +type ServerAttachIsolatedDeviceBase struct { + AutoStart bool `json:"auto_start"` + GpuType string `json:"gpu_type"` + MemoryRequest *int `json:"memory_request"` + SharingMode string `json:"sharing_mode"` + Count *int `json:"count"` +} + +type ServerAttachIsolatedDeviceInput struct { + ServerAttachIsolatedDeviceBase + Device string `json:"device"` + Model string `json:"model"` } type ServerChangeBillingTypeInput struct { diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index 4edc2f4ffd..5651a42cdf 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -19,7 +19,6 @@ import ( cloudmux "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" "yunion.io/x/onecloud/pkg/cloudcommon/types" @@ -776,7 +775,7 @@ type HostUploadGuestsStatusInput struct { } type HostIsolatedDeviceNumaStatsInput struct { - DevType string `json:"dev_type"` + Model string } type GuestUploadContainerStatusResponse struct { diff --git a/pkg/apis/compute/isolated_device.go b/pkg/apis/compute/isolated_device.go index 44f71edf25..5212eae062 100644 --- a/pkg/apis/compute/isolated_device.go +++ b/pkg/apis/compute/isolated_device.go @@ -34,10 +34,13 @@ type IsolateDeviceDetails struct { SIsolatedDevice + MemoryAllocated int + AllocatedCount int + // 云主机名称 - Guest string `json:"guest"` + Guest []string `json:"guest"` // 云主机状态 - GuestStatus string `json:"guest_status"` + GuestStatus []string `json:"guest_status"` } type IsolatedDeviceListInput struct { @@ -95,6 +98,9 @@ type IsolatedDeviceCreateInput struct { // example: GPU DevType string `json:"dev_type"` + // DEVICE sharing mode + SharingMode string `json:"sharing_mode"` + // 设备型号 // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `json:"model"` @@ -137,9 +143,28 @@ type IsolatedDeviceUpdateInput struct { DeviceMinor int `json:"device_minor"` } +type SDelIsolatedDeviceInput struct { + Device string + Index int +} + +type SAddIsolatedDeviceInput struct { + Device string + GpuType string + MemoryRequest *int +} + +type SetIsolatedDeviceInput struct { + AddDevices []SAddIsolatedDeviceInput + DelDevices []SDelIsolatedDeviceInput + AutoStart bool +} + type IsolatedDeviceJsonDesc struct { Id string `json:"id"` DevType string `json:"dev_type"` + GpuType string `json:"gpu_type"` + SharingMode string `json:"sharing_mode"` Model string `json:"model"` Addr string `json:"addr"` VendorDeviceId string `json:"vendor_device_id"` @@ -152,6 +177,8 @@ type IsolatedDeviceJsonDesc struct { MemorySize int `json:"memory_size"` MdevId string `json:"mdev_id"` NumaNode int8 `json:"numa_node"` + MemoryLimit int `json:"memory_limit"` + SmUtilLimit int `json:"sm_util_limit"` } type IsolatedDeviceModelCreateInput struct { @@ -368,3 +395,22 @@ type HostIsolatedDeviceModelDetails struct { HotPluggable bool `json:"hot_pluggable"` DisableAutoDetect bool `json:"disable_auto_detect"` } + +type IsolatedDeviceFilterListInput struct { + IsolateDeviceIds []string `json:"isolate_device_ids"` +} + +type GuestIsolatedDeviceListInput struct { + GuestJointsListInput + + IsolatedDeviceListInput + IsolatedDeviceFilterListInput +} + +type GuestIsolatedDeviceDetails struct { + GuestJointResourceDetails + SGuestIsolatedDevice + SIsolatedDevice + HostResourceInfo + apis.SharableResourceBaseInfo +} diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go index 05d5bff96b..613d5b22af 100644 --- a/pkg/apis/compute/isolated_device_const.go +++ b/pkg/apis/compute/isolated_device_const.go @@ -15,25 +15,83 @@ package compute const ( - DIRECT_PCI_TYPE = "PCI" - GPU_HPC_TYPE = "GPU-HPC" // # for compute - GPU_VGA_TYPE = "GPU-VGA" // # for display - SRIOV_VGPU_TYPE = "SRIOV-VGPU" - LEGACY_VGPU_TYPE = "LEGACY-VGPU" - USB_TYPE = "USB" - NIC_TYPE = "NIC" // nic sriov - NVME_PT_TYPE = "NVME-PT" // nvme passthrough - - NVIDIA_VENDOR_ID = "10de" - AMD_VENDOR_ID = "1002" + DIRECT_PCI_TYPE = "PCI" + GPU_TYPE = "GPU" + USB_TYPE = "USB" + NIC_TYPE = "NIC" // nic sriov + NVME_PT_TYPE = "NVME-PT" // nvme passthrough + NETINT_TYPE = "NETINT" + NPU_TYPE = "NPU" + BINDER_TYPE = "BINDER" ) -var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} -var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE} +var VALID_TYPES = []string{DIRECT_PCI_TYPE, GPU_TYPE, USB_TYPE, NIC_TYPE, NVME_PT_TYPE, NETINT_TYPE, NPU_TYPE, BINDER_TYPE} + +const ( + DEVICE_SHARING_MODE_EXCLUSIVE = "EXCLUSIVE" + DEVICE_SHARING_MODE_SRIOV = "SRIOV" + DEVICE_SHARING_MODE_MPS = "MPS" + DEVICE_SHARING_MODE_HAMI = "HAMI" + DEVICE_SHARING_MODE_UNLIMITED = "UNLIMITED" + DEVICE_SHARING_MODE_MDEV = "MDEV" +) + +var VIRTUAL_SHARING_MODES = []string{DEVICE_SHARING_MODE_HAMI, DEVICE_SHARING_MODE_UNLIMITED, DEVICE_SHARING_MODE_MPS} +var VAILD_SHARING_MODES = []string{DEVICE_SHARING_MODE_EXCLUSIVE, DEVICE_SHARING_MODE_SRIOV, DEVICE_SHARING_MODE_MDEV, DEVICE_SHARING_MODE_HAMI, DEVICE_SHARING_MODE_UNLIMITED, DEVICE_SHARING_MODE_MPS} + +const ( + GPU_HPC = "HPC" + GPU_VGA = "VGA" +) + +const ( + NVIDIA_VENDOR_ID = "10de" + AMD_VENDOR_ID = "1002" + VASTAITECH_VENDOR_ID = "1ec6" +) + +var ID_VENDOR_MAP = map[string]string{ + NVIDIA_VENDOR_ID: "NVIDIA", + AMD_VENDOR_ID: "AMD", + VASTAITECH_VENDOR_ID: "VASTAITECH", +} + +var VENDOR_ID_MAP = map[string]string{ + "NVIDIA": NVIDIA_VENDOR_ID, + "AMD": AMD_VENDOR_ID, +} + +const ( + ISOLATED_DEVICE_MODEL_METADATA_BANDWIDTH = "bandwidth" + ISOLATED_DEVICE_MODEL_METADATA_MEMORY_MB = "memory_mb" + ISOLATED_DEVICE_MODEL_METADATA_TFLOPS = "tflops" +) + +////////////// deprecated + +const ( + GPU_HPC_TYPE = "GPU-HPC" // # for compute + GPU_VGA_TYPE = "GPU-VGA" // # for display + SRIOV_VGPU_TYPE = "SRIOV-VGPU" //done + LEGACY_VGPU_TYPE = "LEGACY-VGPU" //done +) + +var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} //done +var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE} //done + +var GPU_TYPES = []string{ + GPU_HPC_TYPE, GPU_VGA_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE, + CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_VASTAITECH_GPU, +} + +var NETINT_TYPES = []string{ + CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, +} var VALID_CONTAINER_DEVICE_TYPES = []string{ CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, - CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, + CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU, } @@ -49,27 +107,12 @@ var VITRUAL_DEVICE_TYPES = []string{ CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_VASTAITECH_GPU, } -var ID_VENDOR_MAP = map[string]string{ - NVIDIA_VENDOR_ID: "NVIDIA", - AMD_VENDOR_ID: "AMD", -} - -var VENDOR_ID_MAP = map[string]string{ - "NVIDIA": NVIDIA_VENDOR_ID, - "AMD": AMD_VENDOR_ID, -} - -const ( - ISOLATED_DEVICE_MODEL_METADATA_BANDWIDTH = "bandwidth" - ISOLATED_DEVICE_MODEL_METADATA_MEMORY_MB = "memory_mb" - ISOLATED_DEVICE_MODEL_METADATA_TFLOPS = "tflops" -) - func init() { VALID_PASSTHROUGH_TYPES = append(VALID_PASSTHROUGH_TYPES, VALID_CONTAINER_DEVICE_TYPES...) VALID_ATTACH_TYPES = append(VALID_ATTACH_TYPES, VALID_CONTAINER_DEVICE_TYPES...) diff --git a/pkg/apis/compute/zz_generated.model.go b/pkg/apis/compute/zz_generated.model.go index 5dd6228122..d92b06696e 100644 --- a/pkg/apis/compute/zz_generated.model.go +++ b/pkg/apis/compute/zz_generated.model.go @@ -23,6 +23,13 @@ import ( "yunion.io/x/onecloud/pkg/apis/cloudprovider" ) +// IsolatedDeviceAllocateStat is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.IsolatedDeviceAllocateStat. +type IsolatedDeviceAllocateStat struct { + SIsolatedDevice + GuestCount int `json:"guest_count"` + MemoryAllocated int `json:"memory_allocated"` +} + // SAccessGroup is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SAccessGroup. type SAccessGroup struct { apis.SStatusInfrasResourceBase @@ -943,6 +950,10 @@ type SElasticip struct { BgpType string `json:"bgp_type"` // 是否跟随主机删除而自动释放 AutoDellocate *bool `json:"auto_dellocate,omitempty"` + // 下行带宽限制，单位mbps + RxBwLimit int `json:"rx_bw_limit"` + // 上行带宽限制，单位mbps + TxBwLimit int `json:"tx_bw_limit"` } // SExternalProject is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SExternalProject. @@ -1117,6 +1128,23 @@ type SGuest struct { ResourcePool string `json:"resource_pool"` } +// SGuestIsolatedDevice is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestIsolatedDevice. +type SGuestIsolatedDevice struct { + SGuestJointsBase + SIsolatedDeviceResourceBase + // guest isolated device memory size limit + DeviceMemorySize int `json:"device_memory_size"` + // guest isolated device Streaming Multiprocessor Utilization limit + SmUtilLimit int `json:"sm_util_limit"` + // gpu device work type: HPC VGA + GpuType string `json:"gpu_type"` + // guest network index + NetworkIndex int `json:"network_index"` + // guest disk index + DiskIndex byte `json:"disk_index"` + Index byte `json:"index"` +} + // SGuestJointsBase is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestJointsBase. type SGuestJointsBase struct { apis.SVirtualJointResourceBase @@ -1253,6 +1281,10 @@ type SGuestnetwork struct { PortMappings []*GuestPortMapping `json:"port_mappings"` SBillingTypeBase SBillingChargeTypeBase + // 下行带宽限制，单位mbps + RxBwLimit int `json:"rx_bw_limit"` + // 上行带宽限制，单位mbps + TxBwLimit int `json:"tx_bw_limit"` } // SGuestnetworksecgroup is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestnetworksecgroup. @@ -1538,7 +1570,7 @@ type SInstanceSnapshot struct { // 套餐名称 InstanceType string `json:"instance_type"` // 主机快照磁盘容量和 - SizeMb int `json:"size_mb"` + // SizeMb int `nullable:"false" list:"user"` // 镜像ID ImageId string `json:"image_id"` // 是否保存内存 @@ -1590,15 +1622,15 @@ type SIsolatedDevice struct { apis.SExternalizedResourceBase apis.SSharableBaseResource SHostResourceBase - // # PCI / GPU-HPC / GPU-VGA / USB / NIC + // # PCI / GPU / USB / NIC ... // 设备类型 DevType string `json:"dev_type"` + // EXCLUSIVE / SRIOV / MPS / HAMI / SHARE / MIG + SharingMode string `json:"sharing_mode"` + // Device is hot pluggable + HotPluggable bool `json:"hot_pluggable"` // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `json:"model"` - // 云主机Id - GuestId string `json:"guest_id"` - // guest network index - NetworkIndex int `json:"network_index"` // Nic wire id WireId string `json:"wire_id"` // Offload interface name @@ -1607,9 +1639,7 @@ type SIsolatedDevice struct { IsInfinibandNic bool `json:"is_infiniband_nic"` // NVME disk size NvmeSizeMB int `json:"nvme_size_mb"` - // guest disk index - DiskIndex byte `json:"disk_index"` - // # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr` + // # pci address of `Bus:Device.Function` format, or usb bus address of `bus:addr:port` Addr string `json:"addr"` DevicePath string `json:"device_path"` // GPU card path, like /dev/dri/cardX @@ -1655,6 +1685,11 @@ type SIsolatedDevice struct { PcieInfo *IsolatedDevicePCIEInfo `json:"pcie_info"` // device numa node NumaNode byte `json:"numa_node"` + // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, + // or per-slice quota for MPS share mode). 0 means unknown / not applicable. + MemorySize int `json:"memory_size"` + // some of isolated device type support virtual num, like NVIDIA_GPU_SHARE, NVIDIA_MPS + VirtualNum int `json:"virtual_num"` } // SIsolatedDeviceModel is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDeviceModel. @@ -1669,6 +1704,11 @@ type SIsolatedDeviceModel struct { DisableAutoDetect *bool `json:"disable_auto_detect,omitempty"` } +// SIsolatedDeviceResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDeviceResourceBase. +type SIsolatedDeviceResourceBase struct { + IsolatedDeviceId string `json:"isolated_device_id"` +} + // SKafka is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SKafka. type SKafka struct { apis.SVirtualResourceBase @@ -2739,7 +2779,6 @@ type SServerSku struct { apis.SExternalizedResourceBase SCloudregionResourceBase SZoneResourceBase - // SkuId string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"admin_required"` // x2.large InstanceTypeFamily string `json:"instance_type_family"` // x2 InstanceTypeCategory string `json:"instance_type_category"` @@ -2816,18 +2855,15 @@ type SSnapshotPolicy struct { apis.SExternalizedResourceBase SManagedResourceBase SCloudregionResourceBase - // 快照保留天数, -1: 表示永久保留 + // 快照保留天数 RetentionDays int `json:"retention_days"` // 快照保留数量, 优先级高于 RetentionDays, 且仅对本地IDC资源有效 RetentionCount int `json:"retention_count"` // 快照类型, 目前支持 disk, server - // disk: 自动磁盘快照策略， 只能关联磁盘 - // server: 自动主机快照策略, 只能关联主机 Type string `json:"type"` - // 1~7, 1 is Monday, 7 is Sunday + // 1~7, 1 is Monday RepeatWeekdays []int `json:"repeat_weekdays"` - // 0~23, 每小时 - // 创建自动快照策略的时间必须与 RepeatWeekdays 对应的创建周期相一致 + // 0~23 TimePoints []int `json:"time_points"` } diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index f361a4d1d7..c9f4ba50b5 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -93,8 +93,11 @@ type ContainerIsolatedDevice struct { Addr string `json:"addr"` Path string `json:"path"` DeviceType string `json:"device_type"` + SharingMode string `json:"sharing_mode"` CardPath string `json:"card_path"` RenderPath string `json:"render_path"` + MemoryLimit int `json:"memory_limit"` + SmUtilLimit int `json:"sm_util_limit"` Index int `json:"index"` DeviceMinor int `json:"device_minor"` OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index 0fe8abe26c..8f3408eae9 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -849,7 +849,7 @@ func (task *sBaremetalPrepareTask) sendIsolatedDevicesInfo( gpuDevs := make([]isolated_device.IDevice, len(devs)) for i := 0; i < len(devs); i++ { - gpuDevs[i] = isolated_device.NewGPUHPCDevice(devs[i]) + gpuDevs[i] = isolated_device.NewGPUHPCDevice(devs[i], api.DEVICE_SHARING_MODE_EXCLUSIVE) } for _, obj := range objs { diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index 7bc27ef6f5..c5cd585e61 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -513,8 +513,7 @@ func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, e } dev := new(compute.IsolatedDeviceConfig) parts := strings.Split(desc, ":") - devTypes := sets.NewString(compute.VALID_PASSTHROUGH_TYPES...) - devTypes.Insert(compute.VALID_CONTAINER_DEVICE_TYPES...) + devTypes := sets.NewString(compute.VALID_TYPES...) for _, p := range parts { if regutils.MatchUUIDExact(p) { dev.Id = p diff --git a/pkg/compute/container_drivers/device/host.go b/pkg/compute/container_drivers/device/host.go index 0f7a80edd2..223601cd81 100644 --- a/pkg/compute/container_drivers/device/host.go +++ b/pkg/compute/container_drivers/device/host.go @@ -70,7 +70,7 @@ func (h hostDevice) ValidateCreateData(ctx context.Context, userCred mcclient.To return dev, nil } -func (h hostDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) { +func (h hostDevice) ToHostDevice(dev *api.ContainerDevice, guestId string) (*hostapi.ContainerDevice, error) { return &hostapi.ContainerDevice{ Type: apis.CONTAINER_DEVICE_TYPE_HOST, ContainerPath: dev.Host.ContainerPath, diff --git a/pkg/compute/container_drivers/device/isolated_device.go b/pkg/compute/container_drivers/device/isolated_device.go index 38535b0d0c..88ef27d2f4 100644 --- a/pkg/compute/container_drivers/device/isolated_device.go +++ b/pkg/compute/container_drivers/device/isolated_device.go @@ -19,7 +19,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/sets" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -87,7 +86,7 @@ func (i isolatedDevice) ValidateCreateData(ctx context.Context, userCred mcclien return nil, errors.Wrapf(err, "validate create data %s", jsonutils.Marshal(dev)) } isoDev := dev.IsolatedDevice - podDevs, err := pod.GetIsolatedDevices() + podDevs, err := pod.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrap(err, "get isolated devices") } @@ -96,7 +95,7 @@ func (i isolatedDevice) ValidateCreateData(ctx context.Context, userCred mcclien if index >= len(podDevs) { return nil, httperrors.NewInputParameterError("index %d is large than isolated device size %d", index, len(podDevs)) } - isoDev.Id = podDevs[index].GetId() + isoDev.Id = podDevs[index].IsolatedDeviceId // remove index isoDev.Index = nil } else { @@ -104,13 +103,14 @@ func (i isolatedDevice) ValidateCreateData(ctx context.Context, userCred mcclien return nil, httperrors.NewNotEmptyError("id is empty") } foundDisk := false - for _, d := range podDevs { + for i := range podDevs { + d := podDevs[i].GetIsolatedDevice() if d.GetId() == isoDev.Id || d.GetName() == isoDev.Id { isoDev.Id = d.GetId() foundDisk = true - devType := d.DevType - if !sets.NewString(api.VALID_CONTAINER_DEVICE_TYPES...).Has(devType) { - return nil, httperrors.NewInputParameterError("device type %s is not supported by container", devType) + host := d.GetHost() + if host.HostType != api.HOST_TYPE_CONTAINER { + return nil, httperrors.NewInputParameterError("device %s is not supported by container", isoDev.Id) } break } @@ -123,13 +123,17 @@ func (i isolatedDevice) ValidateCreateData(ctx context.Context, userCred mcclien return dev, nil } -func (i isolatedDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) { +func (i isolatedDevice) ToHostDevice(dev *api.ContainerDevice, guestId string) (*hostapi.ContainerDevice, error) { input := dev.IsolatedDevice isoDevObj, err := models.IsolatedDeviceManager.FetchById(input.Id) if err != nil { return nil, errors.Wrapf(err, "Fetch isolated device by id %s", input.Id) } isoDev := isoDevObj.(*models.SIsolatedDevice) + gdev, err := isoDev.GetGuestIsolatedDevice(guestId, input.GuestIsolatedDeviceIndex) + if err != nil { + return nil, errors.Wrap(err, "GetGuestIsolatedDevice") + } return &hostapi.ContainerDevice{ Type: dev.Type, IsolatedDevice: &hostapi.ContainerIsolatedDevice{ @@ -138,7 +142,10 @@ func (i isolatedDevice) ToHostDevice(dev *api.ContainerDevice) (*hostapi.Contain Path: isoDev.DevicePath, CardPath: isoDev.CardPath, DeviceType: isoDev.DevType, + SharingMode: isoDev.SharingMode, RenderPath: isoDev.RenderPath, + MemoryLimit: gdev.DeviceMemorySize, + SmUtilLimit: gdev.SmUtilLimit, Index: isoDev.Index, DeviceMinor: isoDev.DeviceMinor, OnlyEnv: input.OnlyEnv, diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 6e34a43d5a..25e246ec67 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -710,11 +710,11 @@ func (base *SBaseGuestDriver) ValidateGuestHotChangeConfigInput(ctx context.Cont return confs, nil } -func (base *SBaseGuestDriver) BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { +func (base *SBaseGuestDriver) BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SGuestIsolatedDevice) error { return nil } -func (base *SBaseGuestDriver) BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { +func (base *SBaseGuestDriver) BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *models.SGuest, dev *models.SGuestIsolatedDevice) error { return nil } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 6be55cd571..e8b1bef22d 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -851,7 +851,7 @@ func (self *SKVMGuestDriver) CheckMigrate(ctx context.Context, guest *models.SGu } } } - devices, err := guest.GetIsolatedDevices() + devices, err := guest.GetGuestIsolatedDevices() if err != nil { return errors.Wrapf(err, "GetIsolatedDevices") } @@ -879,7 +879,7 @@ func (self *SKVMGuestDriver) CheckLiveMigrate(ctx context.Context, guest *models if cdrom != nil && len(cdrom.ImageId) > 0 { return httperrors.NewBadRequestError("Cannot live migrate with cdrom") } - devices, err := guest.GetIsolatedDevices() + devices, err := guest.GetGuestIsolatedDevices() if err != nil { return errors.Wrapf(err, "GetIsolatedDevices") } diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index fb782f4637..15cd56282b 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -646,12 +646,13 @@ func (drv *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx contex } } - devs, err := guest.GetIsolatedDevices() + devs, err := guest.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrapf(err, "GetIsolatedDevices") } desc.IsolateDevices = []cloudprovider.SIsolateDevice{} - for _, dev := range devs { + for i := range devs { + dev := devs[i].GetIsolatedDevice() desc.IsolateDevices = append(desc.IsolateDevices, cloudprovider.SIsolateDevice{ Id: dev.ExternalId, Name: dev.Name, diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index db0ebc2b75..02a1abf951 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -608,7 +608,7 @@ func (p *SPodDriver) RequestDeleteSnapshot(ctx context.Context, guest *models.SG return p.SKVMGuestDriver.RequestDeleteSnapshot(ctx, guest, task, params) } -func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { +func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, gdev *models.SGuestIsolatedDevice) error { ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId()) if err != nil { return errors.Wrapf(err, "get containers by pod %s", guest.GetId()) @@ -620,10 +620,11 @@ func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mc newDevs := make([]*api.ContainerDevice, 0) releasedDevs := make(map[string]models.ContainerReleasedDevice) for _, curDev := range devs { - if curDev.IsolatedDevice == nil || curDev.IsolatedDevice.Id != dev.GetId() { + if curDev.IsolatedDevice == nil || (curDev.IsolatedDevice.Id != gdev.IsolatedDeviceId && curDev.IsolatedDevice.GuestIsolatedDeviceIndex == int(gdev.Index)) { tmpDev := curDev newDevs = append(newDevs, tmpDev) } else { + dev := gdev.GetIsolatedDevice() releasedDevs[curDev.IsolatedDevice.Id] = *models.NewContainerReleasedDevice(curDev, dev.DevType, dev.Model) } } @@ -640,7 +641,7 @@ func (p *SPodDriver) BeforeDetachIsolatedDevice(ctx context.Context, userCred mc return nil } -func (p *SPodDriver) BeforeAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SIsolatedDevice) error { +func (p *SPodDriver) BeforeAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, dev *models.SGuestIsolatedDevice) error { ctrs, err := models.GetContainerManager().GetContainersByPod(guest.GetId()) if err != nil { return errors.Wrapf(err, "get containers by pod %s", guest.GetId()) @@ -654,7 +655,7 @@ func (p *SPodDriver) BeforeAttachIsolatedDevice(ctx context.Context, userCred mc return nil } -func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrPtr *models.SContainer, dev *models.SIsolatedDevice) error { +func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrPtr *models.SContainer, gdev *models.SGuestIsolatedDevice) error { rlsDevs, err := ctrPtr.GetReleasedDevices(ctx, userCred) if err != nil { return errors.Wrapf(err, "get release devices for container %s", ctrPtr.GetId()) @@ -672,7 +673,7 @@ func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCr if curDev.IsolatedDevice == nil { continue } - if curDev.IsolatedDevice.Id == dev.GetId() { + if curDev.IsolatedDevice.Id == gdev.IsolatedDeviceId && curDev.IsolatedDevice.GuestIsolatedDeviceIndex == int(gdev.Index) { shouldUpdate = false break } @@ -681,7 +682,8 @@ func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCr spec.Devices = append(spec.Devices, &api.ContainerDevice{ Type: apis.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE, IsolatedDevice: &api.ContainerIsolatedDevice{ - Id: dev.GetId(), + Id: gdev.IsolatedDeviceId, + GuestIsolatedDeviceIndex: int(gdev.Index), }, }) if _, err := db.Update(ctrPtr, func() error { @@ -695,6 +697,7 @@ func (p *SPodDriver) attachIsolatedDeviceToContainer(ctx context.Context, userCr if rlsDev.IsolatedDevice == nil { continue } + dev := gdev.GetIsolatedDevice() if rlsDev.DeviceModel == dev.Model && rlsDev.DeviceType == dev.DevType { delete(rlsDevs, id) if err := ctrPtr.SaveReleasedDevices(ctx, userCred, rlsDevs); err != nil { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index e00418607e..323d9d331f 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -119,9 +119,9 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, dev, _ := idev.(*models.SIsolatedDevice) sriovWires = []string{dev.WireId} } else { - wires, err := models.IsolatedDeviceManager.FindUnusedNicWiresByModel(netConfig.SriovDevice.Model) + wires, err := models.IsolatedDeviceManager.FindAvailableNicWiresByModel(netConfig.SriovDevice.Model) if err != nil { - return nil, errors.Wrap(err, "FindUnusedNicWiresByModel") + return nil, errors.Wrap(err, "FindAvailableNicWiresByModel") } sriovWires = wires } diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index 07c84c126c..c425097348 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -869,9 +869,11 @@ func getStorageTypes( } type PCIDevModelTypes struct { - Model string - DevType string - SizeMB int + Model string + DevType string + SharingMode string + NvmeSizeMB int + DevMemorySize int VirtualDev bool Hypervisor string @@ -885,7 +887,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia hostQuery = StorageManager.FilterByOwner(ctx, hostQuery, StorageManager, userCred, ownerId, rbacscope.ScopeDomain) } if len(tenantId) > 0 { - devicesQ = devicesQ.IsNullOrEmpty("guest_id") + devicesQ = IsolatedDeviceManager.GetAvailableIsolatedDeviceQuery(devicesQ) subq := db.SharedResourceManager.Query("resource_id") subq = subq.Equals("resource_type", IsolatedDeviceManager.Keyword()) subq = subq.Equals("target_project_id", tenantId) @@ -916,7 +918,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia devices := devicesQ.SubQuery() hosts := hostQuery.SubQuery() - q := devices.Query(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb")) + q := devices.Query(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("sharing_mode"), devices.Field("nvme_size_mb"), devices.Field("memory_size")) q = q.Filter(sqlchemy.NotIn(devices.Field("dev_type"), []string{api.USB_TYPE, api.NIC_TYPE})) if zone != nil { q = q.Join(hosts, sqlchemy.Equals(devices.Field("host_id"), hosts.Field("id"))) @@ -935,7 +937,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia sqlchemy.IsNullOrEmpty(hosts.Field("manager_id")), )) }*/ - q = q.GroupBy(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb")) + q = q.GroupBy(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("sharing_mode"), devices.Field("nvme_size_mb"), devices.Field("memory_size")) rows, err := q.Rows() if err != nil { @@ -946,20 +948,21 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia gpus := make([]PCIDevModelTypes, 0) gpuModels := make([]string, 0) for rows.Next() { - var m, t string - var sizeMB int + var m, t, sharingMode string + var nvmeSizeMB int + var memSizeMB int var vdev bool var hypervisor string var hostType string - rows.Scan(&hostType, &m, &t, &sizeMB) + rows.Scan(&hostType, &m, &t, &sharingMode, &nvmeSizeMB, &memSizeMB) if m == "" { continue } - if utils.IsInStringArray(t, api.VITRUAL_DEVICE_TYPES) { + if utils.IsInStringArray(sharingMode, api.VIRTUAL_SHARING_MODES) { vdev = true } - if utils.IsInStringArray(t, api.VALID_CONTAINER_DEVICE_TYPES) { + if hostType == api.HOST_TYPE_CONTAINER { hypervisor = api.HYPERVISOR_POD } else { hypervisor = api.HYPERVISOR_KVM @@ -969,7 +972,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia hypervisor = api.HYPERVISOR_ZETTAKIT } - gpus = append(gpus, PCIDevModelTypes{m, t, sizeMB, vdev, hypervisor}) + gpus = append(gpus, PCIDevModelTypes{m, t, sharingMode, nvmeSizeMB, memSizeMB, vdev, hypervisor}) if !utils.IsInStringArray(m, gpuModels) { gpuModels = append(gpuModels, m) diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index f98ab0af4c..98a3df22b3 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -1294,35 +1294,31 @@ func (self *SGuest) SyncVMIsolateDevices(ctx context.Context, userCred mcclient. if err != nil { return err } - devs, err := self.GetIsolatedDevices() + gdevs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrapf(err, "GetIsolatedDevices") } result := compare.SyncResult{} - for i := range devs { - if !utils.IsInStringArray(devs[i].ExternalId, externalIds) { - _, err = db.Update(&devs[i], func() error { - devs[i].GuestId = "" - return nil - }) + for i := range gdevs { + dev := gdevs[i].GetIsolatedDevice() + if !utils.IsInStringArray(dev.ExternalId, externalIds) { + err = gdevs[i].Detach(ctx, userCred) if err != nil { return err } result.Delete() } } - devs = []SIsolatedDevice{} + devs := []SIsolatedDevice{} sq := HostManager.Query("id").Equals("manager_id", host.ManagerId).SubQuery() q := IsolatedDeviceManager.Query().In("host_id", sq).In("external_id", externalIds) err = db.FetchModelObjects(IsolatedDeviceManager, q, &devs) if err != nil { return err } + for i := range devs { - _, err = db.Update(&devs[i], func() error { - devs[i].GuestId = self.Id - return nil - }) + err = self.attachIsolatedDevice(ctx, userCred, &devs[i], nil, nil, nil, "") if err != nil { return err } diff --git a/pkg/compute/models/container_drivers.go b/pkg/compute/models/container_drivers.go index fac10ad6a5..d639bfa00a 100644 --- a/pkg/compute/models/container_drivers.go +++ b/pkg/compute/models/container_drivers.go @@ -134,7 +134,7 @@ type IContainerDeviceDriver interface { GetType() apis.ContainerDeviceType ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, dev *api.ContainerDevice, input *api.ServerCreateInput) error ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) - ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) + ToHostDevice(dev *api.ContainerDevice, guestId string) (*hostapi.ContainerDevice, error) } type IContainerLifecyleDriver interface { diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index 9ede8be491..d7ad5268ce 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -944,7 +944,7 @@ func (c *SContainer) ToHostContainerSpec(ctx context.Context, userCred mcclient. } ctrDevs := make([]*hostapi.ContainerDevice, 0) for _, dev := range c.Spec.Devices { - ctrDev, err := GetContainerDeviceDriver(dev.Type).ToHostDevice(dev) + ctrDev, err := GetContainerDeviceDriver(dev.Type).ToHostDevice(dev, c.GuestId) if err != nil { return nil, errors.Wrapf(err, "ToHostDevice %s", jsonutils.Marshal(dev)) } diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 715d76eaf1..69d5375d23 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -1178,7 +1178,7 @@ func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, us if err != nil { return nil, errors.Wrap(err, "unmarshal virtual dev configs") } - devs, err := self.GetIsolatedDevices() + devs, err := self.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrap(err, "GetIsolatedDevices") } @@ -1689,11 +1689,11 @@ func (self *SGuest) StartInsertVfdTask(ctx context.Context, floppyOrdinal int64, } func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) error { - devs, err := self.GetIsolatedDevices() + guestDevs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrap(err, "guest get isolated devices") } - if len(devs) == 0 { + if len(guestDevs) == 0 { return nil } host, err := self.GetHost() @@ -1701,16 +1701,19 @@ func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCre return errors.Wrap(err, "guest get host") } var numaNodeBalance = true - detachDevs := make([]SIsolatedDevice, 0) + detachDevs := make([]SGuestIsolatedDevice, 0) originDevConfigs := make([]api.IsolatedDeviceConfig, 0) - for i := range devs { - if utils.IsInStringArray(devs[i].DevType, api.VITRUAL_DEVICE_TYPES) { + for i := range guestDevs { + dev := guestDevs[i].GetIsolatedDevice() + if dev.SharingMode == api.DEVICE_SHARING_MODE_UNLIMITED { originDevConfigs = append(originDevConfigs, api.IsolatedDeviceConfig{ - Model: devs[i].Model, - DevType: devs[i].DevType, + Model: dev.Model, + DevType: dev.DevType, + MemoryRequest: guestDevs[i].DeviceMemorySize, + SmUtilLimit: guestDevs[i].SmUtilLimit, }) - detachDevs = append(detachDevs, devs[i]) - isBalance, err := host.VirtualDeviceNumaBalance(devs[i].DevType, devs[i].NumaNode) + detachDevs = append(detachDevs, guestDevs[i]) + isBalance, err := host.VirtualDeviceNumaBalance(dev.Model, dev.NumaNode) if err != nil { return errors.Wrap(err, "VirtualDeviceNumaBalance") } @@ -1728,20 +1731,24 @@ func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCre lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) + detachedDevConfigs := []*api.IsolatedDeviceConfig{} for i := 0; i < len(detachDevs); i++ { err := self.detachIsolateDevice(ctx, userCred, &detachDevs[i]) if err != nil { return errors.Wrapf(err, "detach device %s", detachDevs[i].GetId()) } + dev := detachDevs[i].GetIsolatedDevice() + detachedDevConfigs = append(detachedDevConfigs, &api.IsolatedDeviceConfig{ + Model: dev.Model, + DevType: dev.DevType, + MemoryRequest: detachDevs[i].DeviceMemorySize, + SmUtilLimit: detachDevs[i].SmUtilLimit, + }) } usedDeviceMap := map[string]*SIsolatedDevice{} - for i := range detachDevs { - devConfig := &api.IsolatedDeviceConfig{ - Model: detachDevs[i].Model, - DevType: detachDevs[i].DevType, - } - err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil) + for i := range detachedDevConfigs { + err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, detachedDevConfigs[i], userCred, usedDeviceMap, nil) if err != nil { return errors.Wrap(err, "attachHostDeviceToGuestByModel") } @@ -2347,12 +2354,16 @@ func (self *SGuest) GetReleasedIsolatedDevices(ctx context.Context, userCred mcc return devs, nil } -func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { +func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SGuestIsolatedDevice) error { records := make([]api.ServerReleasedIsolatedDevice, 0) - for _, dev := range devs { + for i := range devs { + dev := devs[i].GetIsolatedDevice() record := api.ServerReleasedIsolatedDevice{ - DevType: dev.DevType, - Model: dev.Model, + DevType: dev.DevType, + Model: dev.Model, + GpuType: devs[i].GpuType, + SharingMode: dev.SharingMode, + MemoryRequest: devs[i].DeviceMemorySize, } records = append(records, record) } @@ -2362,14 +2373,14 @@ func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcc return nil } -func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { +func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SGuestIsolatedDevice) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) for i := 0; i < len(devs); i++ { // check first - dev := devs[i] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { + dev := devs[i].GetIsolatedDevice() + if !dev.IsValidAttachDev() { if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) @@ -2414,7 +2425,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc } var detachAllDevice = jsonutils.QueryBoolean(data, "detach_all", false) - devs := make([]SIsolatedDevice, 0) + devs := make([]SGuestIsolatedDevice, 0) if !detachAllDevice { device, err := data.GetString("device") if err != nil { @@ -2422,6 +2433,12 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError("%s", msg) } + index, err := data.Int("index") + if err != nil { + msg := "Missing isolated device index" + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError("%s", msg) + } iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -2429,9 +2446,16 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError(msgFmt, device) } - devs = append(devs, *iDev.(*SIsolatedDevice)) + dev := iDev.(*SIsolatedDevice) + gdev, err := dev.GetGuestIsolatedDevice(self.Id, int(index)) + if err != nil { + msg := err.Error() + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError("%s", msg) + } + devs = append(devs, *gdev) } else { - devs, _ = self.GetIsolatedDevices() + devs, _ = self.GetGuestIsolatedDevices() } if err := self.DetachIsolatedDevices(ctx, userCred, devs); err != nil { return nil, err @@ -2442,7 +2466,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") } -func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string, index int) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -2451,25 +2475,28 @@ func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, user return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - return self.DetachIsolatedDevices(ctx, userCred, []SIsolatedDevice{*dev}) + gdev, err := dev.GetGuestIsolatedDevice(self.Id, index) + if err != nil { + msgFmt := "Isolated device %s index %d not found" + msg := fmt.Sprintf(msgFmt, device, index) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return httperrors.NewBadRequestError(msgFmt, device) + } + return self.DetachIsolatedDevices(ctx, userCred, []SGuestIsolatedDevice{*gdev}) } -func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice) error { - if dev.GuestId != self.Id { +func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, gdev *SGuestIsolatedDevice) error { + if gdev.GuestId != self.Id { msg := "Isolated device is not attached to this guest" logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return httperrors.NewBadRequestError("%s", msg) } + dev := gdev.GetIsolatedDevice() drv, _ := self.GetDriver() - if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, dev); err != nil { - return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, gdev); err != nil { + return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(gdev), self.GetId()) } - _, err := db.Update(dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - dev.DiskIndex = -1 - return nil - }) + err := gdev.Detach(ctx, userCred) if err != nil { return err } @@ -2478,7 +2505,7 @@ func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.T } // 挂载透传设备 -func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.ServerAttachIsolatedDeviceInput) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } @@ -2489,20 +2516,29 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc return nil, httperrors.NewInvalidStatusError("%s", msg) } var err error - autoStart := jsonutils.QueryBoolean(data, "auto_start", false) - if data.Contains("device") { - device, _ := data.GetString("device") - err = self.StartAttachIsolatedDeviceGpuOrUsb(ctx, userCred, device, autoStart) - } else if data.Contains("model") { - vmodel, _ := data.GetString("model") - var count int64 = 1 - if data.Contains("count") { - count, _ = data.Int("count") + if input.GpuType != "" && !utils.IsInStringArray(input.GpuType, []string{api.GPU_VGA, api.GPU_HPC}) { + return nil, httperrors.NewInputParameterError("gpu_type %s not vaild", input.GpuType) + } + if input.MemoryRequest != nil && *input.MemoryRequest <= 0 { + return nil, httperrors.NewBadRequestError("guest attach gpu memory request count must > 0") + } + + if input.Device != "" { + err = self.StartAttachIsolatedDeviceGpuOrUsb(ctx, userCred, input.Device, input.GpuType, input.MemoryRequest, input.AutoStart) + } else if input.Model != "" { + if !utils.IsInStringArray(input.SharingMode, api.VAILD_SHARING_MODES) { + return nil, httperrors.NewInputParameterError("shaing_mode %s not valid", input.SharingMode) + } + + var count int = 1 + if input.Count != nil { + count = *input.Count } if count < 1 { return nil, httperrors.NewBadRequestError("guest attach gpu count must > 0") } - err = self.StartAttachIsolatedDevices(ctx, userCred, vmodel, int(count), autoStart) + input.Count = &count + err = self.StartAttachIsolatedDevices(ctx, userCred, input.Model, input.GpuType, input.SharingMode, count, input.MemoryRequest, input.AutoStart) } else { return nil, httperrors.NewMissingParameterError("device||model") } @@ -2513,101 +2549,104 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc return nil, nil } -func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int, autoStart bool) error { - if err := self.startAttachIsolatedDevices(ctx, userCred, devModel, count); err != nil { +func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel, gpuType, sharingMode string, count int, memoryRequest *int, autoStart bool) error { + if err := self.startAttachIsolatedDevices(ctx, userCred, devModel, gpuType, sharingMode, count, memoryRequest); err != nil { return err } // perform post attach task return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int) error { +func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int, gpuType, sharingMode string, memoryRequest *int) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) + attachedGpus, err := self.GetGuestIsolatedDevices() + if err != nil { + return errors.Wrap(err, "get isolated devices") + } + attachedAddrs := map[string]struct{}{} + for i := range attachedGpus { + dev := attachedGpus[i].GetIsolatedDevice() + attachedAddrs[dev.Addr] = struct{}{} + gdev := attachedGpus[i].GetIsolatedDevice() + if sharingMode == api.DEVICE_SHARING_MODE_MDEV { + if gdev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") + } else if gdev.DevType == api.GPU_TYPE { + return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") + } + } + } - unusedDevs := []SIsolatedDevice{} for devModel, count := range devModelCount { - devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count) + devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode) if err != nil { return httperrors.NewInternalServerError("fetch gpu failed %s", err) } - if len(devs) == 0 || len(devs) != count { + if len(devs) == 0 { return httperrors.NewBadRequestError("required %d %s isolated devices on host %s, but not enough are available", count, devModel, host.GetName()) } - dev := devs[0] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) + avaDevs := make([]SIsolatedDevice, 0) + avaDevCnt := 0 + for i := range devs { + dev := devs[i] + if _, ok := attachedAddrs[dev.Addr]; ok { + continue + } + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if memoryRequest == nil || *memoryRequest <= 0 { + return httperrors.NewBadRequestError("dev sharing_mode %s memory request invalid", dev.SharingMode) + } + if dev.IsEnough(*memoryRequest) { + avaDevs = append(avaDevs, dev) + avaDevCnt += 1 + } } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) + cnt, err := dev.getAllocatedCount() + if err != nil { + return errors.Wrap(err, "getAllocatedCount") + } + if dev.VirtualNum > cnt { + avaCnt := dev.VirtualNum - cnt + avaDevCnt += avaCnt + for j := 0; j < avaCnt; j++ { + avaDevs = append(avaDevs, dev) + } } } } - if dev.DevType == api.LEGACY_VGPU_TYPE { - attachedGpus, err := self.GetIsolatedDevices() - if err != nil { - return errors.Wrap(err, "get isolated devices") - } - for i := range attachedGpus { - if attachedGpus[i].DevType == api.LEGACY_VGPU_TYPE { - return httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1") - } else if utils.IsInStringArray(attachedGpus[i].DevType, api.VALID_GPU_TYPES) { - return httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus") - } - } - } else if dev.DevType == api.CONTAINER_DEV_NVIDIA_MPS { - allDevs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, -1) - if err != nil { - return httperrors.NewInternalServerError("fetch gpu failed %s", err) - } - attachedGpus, err := self.GetIsolatedDevices() - if err != nil { - return httperrors.NewInternalServerError("get attached isolated devices %s", err) - } - attachedAddrs := map[string]struct{}{} - for i := range attachedGpus { - addr := strings.Split(attachedGpus[i].Addr, "-")[0] - attachedAddrs[addr] = struct{}{} - } - validDevs := []SIsolatedDevice{} - for i := range allDevs { - devAddr := strings.Split(allDevs[i].Addr, "-")[0] - if _, ok := attachedAddrs[devAddr]; ok { - continue - } - validDevs = append(validDevs, allDevs[i]) - } - if len(validDevs) < count { - return httperrors.NewInsufficientResourceError("required %d %s isolated devices on host %s, but not enough are available", count, devModel, host.GetName()) - } - devs = validDevs[:count] + if avaDevCnt < count { + return httperrors.NewInsufficientResourceError("Available device count %d less then request count %d", avaDevCnt, count) } - unusedDevs = append(unusedDevs, devs...) - } - defer func() { go host.ClearSchedDescCache() }() - for i := 0; i < len(unusedDevs); i++ { - if err := self.attachIsolatedDevice(ctx, userCred, &unusedDevs[i], nil, nil); err != nil { - return errors.Wrapf(err, "attach device %s", unusedDevs[i].GetId()) + dev := avaDevs[0] + if !dev.HotPluggable && self.GetStatus() == api.VM_RUNNING { + return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, dev.Model) + } + + for k := 0; k < count; k++ { + if err := self.attachIsolatedDevice(ctx, userCred, &avaDevs[k], nil, nil, memoryRequest, gpuType); err != nil { + return errors.Wrapf(err, "attach device %s", avaDevs[k].GetId()) + } } } + go host.ClearSchedDescCache() return nil } -func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error { - return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count}) +func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel, gpuType, sharingMode string, count int, memoryRequest *int) error { + return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count}, gpuType, sharingMode, memoryRequest) } -func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { - if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device); err != nil { +func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device, gpuType string, memoryRequest *int, autoStart bool) error { + if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device, gpuType, memoryRequest); err != nil { return err } // perform post attach task return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device, gpuType string, memoryRequest *int) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -2616,29 +2655,45 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - } + gdevs, err := self.GetGuestIsolatedDevices() + if err != nil { + return err + } + for i := range gdevs { + if gdevs[i].IsolatedDeviceId == dev.Id { + return httperrors.NewBadRequestError("device %s is already in used by guest %s", dev.Id, self.GetName()) } } - if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { - return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus()) + if !dev.HotPluggable && self.GetStatus() == api.VM_RUNNING { + return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, dev.Model) } - if dev.DevType == api.LEGACY_VGPU_TYPE { - devs, err := self.GetIsolatedDevices() + if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { + return httperrors.NewInvalidStatusError("Can't attach isolated device when status is %q", self.GetStatus()) + } + + reqCnt := 1 + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if memoryRequest == nil || *memoryRequest <= 0 { + return httperrors.NewBadRequestError("Dev sharing_mode %s memory request invalid", dev.SharingMode) + } + reqCnt = *memoryRequest + } + if !dev.IsEnough(reqCnt) { + return httperrors.NewBadRequestError("Dev %s is not enough", dev.GetName()) + } + + if dev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + devs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrap(err, "get isolated devices") } for i := range devs { - if devs[i].DevType == api.LEGACY_VGPU_TYPE { - return httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1") - } else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) { - return httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus") + gdev := devs[i].GetIsolatedDevice() + if gdev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") + } else if gdev.DevType == api.GPU_TYPE { + return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") } } } @@ -2646,7 +2701,7 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - err = self.attachIsolatedDevice(ctx, userCred, dev, nil, nil) + err = self.attachIsolatedDevice(ctx, userCred, dev, nil, nil, memoryRequest, gpuType) var msg string if err != nil { msg = err.Error() @@ -2657,39 +2712,52 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred return err } -func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice, networkIndex *int, diskIndex *int8) error { - if len(dev.GuestId) > 0 { - return fmt.Errorf("Isolated device already attached to another guest: %s", dev.GuestId) +func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice, networkIndex *int, diskIndex *int8, memoryRequest *int, gpuType string) error { + if dev.IsFull() { + return fmt.Errorf("Isolated device already allocated") } if dev.HostId != self.HostId { return fmt.Errorf("Isolated device and guest are not located in the same host") } + + lockman.LockObject(ctx, self) + defer lockman.ReleaseObject(ctx, self) + // guest isolated device attach + guestIsolatedDevice := SGuestIsolatedDevice{} + guestIsolatedDevice.SetModelManager(GuestIsolatedDeviceManager, &guestIsolatedDevice) + guestIsolatedDevice.GuestId = self.Id + guestIsolatedDevice.IsolatedDeviceId = dev.Id + guestIsolatedDevice.Index = self.getIsolatedDeviceIndex() + + if networkIndex != nil { + guestIsolatedDevice.NetworkIndex = *networkIndex + } + if diskIndex != nil { + guestIsolatedDevice.DiskIndex = *diskIndex + } + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI && memoryRequest != nil { + guestIsolatedDevice.DeviceMemorySize = *memoryRequest + } + if utils.IsInStringArray(gpuType, []string{api.GPU_HPC, api.GPU_VGA}) { + guestIsolatedDevice.GpuType = gpuType + } + drv, _ := self.GetDriver() - if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, dev); err != nil { - return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, &guestIsolatedDevice); err != nil { + return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(guestIsolatedDevice), self.GetId()) } - if _, err := db.Update(dev, func() error { - dev.GuestId = self.Id - if networkIndex != nil { - dev.NetworkIndex = *networkIndex - } else { - dev.NetworkIndex = -1 - } - if diskIndex != nil { - dev.DiskIndex = *diskIndex - } else { - dev.DiskIndex = -1 - } - return nil - }); err != nil { - return errors.Wrap(err, "db.Update") + + err := GuestIsolatedDeviceManager.TableSpec().Insert(ctx, &guestIsolatedDevice) + if err != nil { + return err } + db.OpsLog.LogEvent(self, db.ACT_GUEST_ATTACH_ISOLATED_DEVICE, dev.GetShortDesc(ctx), userCred) return nil } // 设置透传设备 -func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.SetIsolatedDeviceInput) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } @@ -2697,46 +2765,27 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli (self.Hypervisor == api.HYPERVISOR_POD && self.GetStatus() != api.VM_READY) { return nil, httperrors.NewInvalidStatusError("Can't set isolated device when guest is %s", self.GetStatus()) } - var addDevs []string - { - addDevices, err := data.Get("add_devices") - if err == nil { - arrAddDev, ok := addDevices.(*jsonutils.JSONArray) - if ok { - addDevs = arrAddDev.GetStringArray() - } else { - return nil, httperrors.NewInputParameterError("attach devices is not string array") - } - } - } - - var delDevs []string - { - delDevices, err := data.Get("del_devices") - if err == nil { - arrDelDev, ok := delDevices.(*jsonutils.JSONArray) - if ok { - delDevs = arrDelDev.GetStringArray() - } else { - return nil, httperrors.NewInputParameterError("detach devices is not string array") - } - } - } + var addDevs = input.AddDevices + var delDevs = input.DelDevices // detach first for i := 0; i < len(delDevs); i++ { - err := self.startDetachIsolateDeviceWithoutNic(ctx, userCred, delDevs[i]) + err := self.startDetachIsolateDeviceWithoutNic(ctx, userCred, delDevs[i].Device, delDevs[i].Index) if err != nil { return nil, err } } for i := 0; i < len(addDevs); i++ { - err := self.startAttachIsolatedDevGeneral(ctx, userCred, addDevs[i]) + if addDevs[i].GpuType != "" && !utils.IsInStringArray(addDevs[i].GpuType, []string{api.GPU_VGA, api.GPU_HPC}) { + return nil, httperrors.NewInputParameterError("gpu_type %s not vaild", addDevs[i].GpuType) + } + + err := self.startAttachIsolatedDevGeneral(ctx, userCred, addDevs[i].Device, addDevs[i].GpuType, addDevs[i].MemoryRequest) if err != nil { return nil, err } } - return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") + return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, input.AutoStart, "") } func (self *SGuest) StartIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parentId string) error { @@ -3569,7 +3618,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T func (self *SGuest) ChangeConfToSchedDesc(addCpu, addExtraCpu, addMem int, schedInputDisks []*api.DiskConfig) *schedapi.ScheduleInput { region, _ := self.GetRegion() - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() desc := &schedapi.ScheduleInput{ ServerConfig: schedapi.ServerConfig{ ServerConfigs: &api.ServerConfigs{ @@ -4644,7 +4693,7 @@ func (self *SGuest) PerformCreateBackup( if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewBadRequestError("Backup only support hypervisor kvm") } - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() if len(devs) > 0 { return nil, httperrors.NewBadRequestError("Cannot create backup with isolated devices") } @@ -5264,9 +5313,10 @@ func (self *SGuest) GenerateVirtInstallCommandLine( } // isolated devices - isolatedDevices, _ := self.GetIsolatedDevices() + isolatedDevices, _ := self.GetGuestIsolatedDevices() for _, isolatedDev := range isolatedDevices { - cmd += L(fmt.Sprintf("--hostdev %s", isolatedDev.Addr)) + dev := isolatedDev.GetIsolatedDevice() + cmd += L(fmt.Sprintf("--hostdev %s", dev.Addr)) } if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_BLOCK_STREAM}) { @@ -5532,7 +5582,7 @@ func (self *SGuest) validateForBatchMigrate(ctx context.Context, rescueMode bool if len(guest.BackupHostId) > 0 { return guest, httperrors.NewBadRequestError("guest %s has backup, can't migrate", guest.Name) } - devs, _ := guest.GetIsolatedDevices() + devs, _ := guest.GetGuestIsolatedDevices() if len(devs) > 0 { return guest, httperrors.NewBadRequestError("guest %s has isolated device, can't migrate", guest.Name) } @@ -5930,7 +5980,6 @@ func (self *SGuest) StartSnapshotResetTask(ctx context.Context, userCred mcclien data.Add(jsonutils.NewBool(withMemory), "with_memory") self.SetStatus(ctx, userCred, api.VM_START_SNAPSHOT_RESET, "start snapshot reset task") instanceSnapshot.SetStatus(ctx, userCred, api.INSTANCE_SNAPSHOT_RESET, "start snapshot reset task") - log.Errorf("====data: %s", data) if task, err := taskman.TaskManager.NewTask( ctx, "InstanceSnapshotResetTask", instanceSnapshot, userCred, data, "", "", nil, ); err != nil { @@ -6969,7 +7018,7 @@ func (self *SGuest) PerformProbeIsolatedDevices(ctx context.Context, userCred mc return nil, errors.Wrapf(err, "FetchById %q", id) } dev := devObj.(*SIsolatedDevice) - if dev.GuestId == "" { + if !dev.IsFull() { devs = append(devs, dev) } } @@ -7107,12 +7156,13 @@ func (self *SGuest) GetDetailsCpusetCores(ctx context.Context, userCred mcclient func (self *SGuest) GetDetailsNumaInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *api.ServerGetNumaInfoInput) (*api.ServerGetNumaInfoResp, error) { ret := new(api.ServerGetNumaInfoResp) - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() if len(devs) > 0 { ret.IsolatedDevicesNumaNode = make([]int8, 0) for i := range devs { - if devs[i].NumaNode > 0 { - ret.IsolatedDevicesNumaNode = append(ret.IsolatedDevicesNumaNode, devs[i].NumaNode) + dev := devs[i].GetIsolatedDevice() + if dev.NumaNode > 0 { + ret.IsolatedDevicesNumaNode = append(ret.IsolatedDevicesNumaNode, dev.NumaNode) } } } @@ -7178,12 +7228,13 @@ func (self *SGuest) GetDetailsHardwareInfo(ctx context.Context, userCred mcclien } // fill GPU info - devs, err := self.GetIsolatedDevices() + devs, err := self.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrap(err, "get isolated devices") } gpuInfos := make([]*api.ServerHardwareInfoGPU, 0) - for _, dev := range devs { + for i := range devs { + dev := devs[i].GetIsolatedDevice() if !dev.IsGPU() { continue } diff --git a/pkg/compute/models/guest_isolated_devices.go b/pkg/compute/models/guest_isolated_devices.go new file mode 100644 index 0000000000..9ca152bead --- /dev/null +++ b/pkg/compute/models/guest_isolated_devices.go @@ -0,0 +1,316 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// +onecloud:swagger-gen-ignore +type SGuestIsolatedDeviceManager struct { + SGuestJointsManager + SIsolatedDeviceResourceBaseManager +} + +var GuestIsolatedDeviceManager *SGuestIsolatedDeviceManager + +func init() { + db.InitManager(func() { + GuestIsolatedDeviceManager = &SGuestIsolatedDeviceManager{ + SGuestJointsManager: NewGuestJointsManager( + SGuestIsolatedDevice{}, + "guestisolateddevices_tbl", + "guestisolateddevice", + "guestisolateddevices", + IsolatedDeviceManager, + ), + } + GuestIsolatedDeviceManager.SetVirtualObject(GuestIsolatedDeviceManager) + GuestIsolatedDeviceManager.TableSpec().AddIndex(false, "isolated_device_id", "guest_id") + }) +} + +// +onecloud:model-api-gen +type SGuestIsolatedDevice struct { + SGuestJointsBase + SIsolatedDeviceResourceBase + + // guest isolated device memory size limit + DeviceMemorySize int `nullable:"true" default:"0" list:"user" update:"user" create:"optional"` + // guest isolated device Streaming Multiprocessor Utilization limit + SmUtilLimit int `nullable:"true" default:"0" list:"user" update:"user" create:"optional"` + // gpu device work type: HPC VGA + GpuType string `width:"16" charset:"ascii" nullable:"true" default:"" index:"true" list:"user" create:"optional" update:"user"` + + // guest network index + NetworkIndex int `nullable:"true" default:"-1" list:"user"` + // guest disk index + DiskIndex int8 `nullable:"true" default:"-1" list:"user"` + + Index int8 `nullable:"false" default:"0" list:"user"` +} + +func (manager *SGuestIsolatedDeviceManager) GetSlaveFieldName() string { + return "isolated_device_id" +} + +func (self *SGuestIsolatedDevice) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, self) +} + +func (self *SGuestIsolatedDevice) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DetachJoint(ctx, userCred, self) +} + +func (manager *SGuestIsolatedDeviceManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = manager.SGuestJointsManager.ListItemFilter(ctx, q, userCred, query.GuestJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemFilter") + } + if len(query.IsolateDeviceIds) > 0 { + query.IsolatedDeviceListInput.Ids = query.IsolateDeviceIds + } + q, err = manager.SIsolatedDeviceResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemFilter") + } + return q, nil +} + +func (manager *SGuestIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SGuestJointsManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemExportKeys") + } + if keys.ContainsAny(manager.SIsolatedDeviceResourceBaseManager.GetExportKeys()...) { + q, err = manager.SIsolatedDeviceResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SIsolatedDeviceManager.ListItemExportKeys") + } + } + + return q, nil +} + +func (manager *SGuestIsolatedDeviceManager) CreateByInsertOrUpdate() bool { + return false +} + +func (manager *SGuestIsolatedDeviceManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.GuestIsolatedDeviceDetails { + rows := make([]api.GuestIsolatedDeviceDetails, len(objs)) + guestRows := manager.SGuestJointsManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + isolatedDeviceRows := manager.SIsolatedDeviceResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for i := range rows { + rows[i].GuestJointResourceDetails = guestRows[i] + rows[i].SIsolatedDevice = isolatedDeviceRows[i].SIsolatedDevice + rows[i].HostResourceInfo = isolatedDeviceRows[i].HostResourceInfo + rows[i].SharableResourceBaseInfo = isolatedDeviceRows[i].SharableResourceBaseInfo + } + + return rows +} + +func (manager *SGuestIsolatedDeviceManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SGuestJointsManager.OrderByExtraFields(ctx, q, userCred, query.GuestJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.OrderByExtraFields") + } + q, err = manager.SIsolatedDeviceResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query) + if err != nil { + return nil, errors.Wrap(err, "SNetworkResourceBaseManager.OrderByExtraFields") + } + + return q, nil +} + +func (dev *SIsolatedDevice) getAllocatedMemorySize() (int, error) { + sq := GuestIsolatedDeviceManager.Query(). + Equals("isolated_device_id", dev.Id). + SubQuery() + + q := sq.Query( + sqlchemy.SUM("memory_used", sq.Field("device_memory_size")), + ) + + var result struct { + MemoryUsed int + } + + err := q.First(&result) + if err != nil { + return 0, err + } + return result.MemoryUsed, nil +} + +func (dev *SIsolatedDevice) getAllocatedCount() (int, error) { + return GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", dev.Id).CountWithError() +} + +func (dev *SIsolatedDevice) getAttachedGuestIds() []string { + q := GuestIsolatedDeviceManager.Query("guest_id").Equals("isolated_device_id", dev.Id) + type sGuestId struct { + GuestId string + } + + guestIsolatedDevices := make([]sGuestId, 0) + err := q.All(&guestIsolatedDevices) + if err != nil { + log.Errorf("failed get guest isolated devices %s", err) + return nil + } + ret := make([]string, len(guestIsolatedDevices)) + for i := range guestIsolatedDevices { + ret[i] = guestIsolatedDevices[i].GuestId + } + return ret +} + +func (dev *SIsolatedDevice) getAttachedGuests() []SGuestIsolatedDevice { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", dev.Id) + guestIsolatedDevices := make([]SGuestIsolatedDevice, 0) + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &guestIsolatedDevices) + if err != nil { + log.Errorf("failed get guest isolated devices %s", err) + return nil + } + return guestIsolatedDevices +} + +func (guest *SGuest) GetIsolatedDevicesQuery() *sqlchemy.SQuery { + return GuestIsolatedDeviceManager.Query().Equals("guest_id", guest.Id) +} + +func (guest *SGuest) GetGuestIsolatedDevices() ([]SGuestIsolatedDevice, error) { + gdevs := make([]SGuestIsolatedDevice, 0) + q := guest.GetIsolatedDevicesQuery().Asc("index") + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return gdevs, nil +} + +func (guest *SGuest) GetGuestGpuIsolatedDevices() ([]SGuestIsolatedDevice, error) { + gdevs := make([]SGuestIsolatedDevice, 0) + q := guest.GetIsolatedDevicesQuery() + isq := IsolatedDeviceManager.Query() + + cond := sqlchemy.Equals(isq.Field("dev_type"), api.GPU_TYPE) + isq = isq.Filter(cond) + sidq := isq.SubQuery() + q = q.Join(sidq, sqlchemy.Equals(q.Field("isolated_device_id"), sidq.Field("id"))) + q = q.Asc("index") + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return gdevs, nil +} + +func (self *SGuest) getIsolatedDeviceIndex() int8 { + guestDevs, _ := self.GetGuestIsolatedDevices() + var max uint + for i := 0; i < len(guestDevs); i++ { + if uint(guestDevs[i].Index) > max { + max = uint(guestDevs[i].Index) + } + } + + idxs := make([]int, max+1) + for i := 0; i < len(guestDevs); i++ { + idxs[guestDevs[i].Index] = 1 + } + + // find first idx not set + for i := 0; i < len(idxs); i++ { + if idxs[i] != 1 { + return int8(i) + } + } + + return int8(max + 1) +} + +func (self *SGuestIsolatedDevice) GetIsolatedDevice() *SIsolatedDevice { + dev, err := IsolatedDeviceManager.FetchById(self.IsolatedDeviceId) + if err != nil { + log.Errorf("IsolatedDeviceManager.FetchById %s", err) + return nil + } + return dev.(*SIsolatedDevice) +} + +func (self *SIsolatedDevice) GetGuestIsolatedDevice(guestId string, index int) (*SGuestIsolatedDevice, error) { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", self.Id).Equals("guest_id", guestId).Equals("index", index) + ret := &SGuestIsolatedDevice{} + err := q.First(ret) + if err != nil { + return nil, errors.Wrapf(err, "fetch guest %s isolated device by id %s index %d", guestId, self.Id, index) + } + ret.SetModelManager(GuestIsolatedDeviceManager, ret) + return ret, nil +} + +func (self *SIsolatedDevice) GetAllGuestIsolatedDevices() ([]SGuestIsolatedDevice, error) { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", self.Id) + ret := []SGuestIsolatedDevice{} + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &ret) + if err != nil { + return nil, errors.Wrapf(err, "fetch guest isolated device by id %s", self.Id) + } + return ret, nil +} + +func (self *SGuestIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { + dev := self.GetIsolatedDevice() + devDesc := dev.getDesc() + devDesc.DiskIndex = self.DiskIndex + devDesc.NetworkIndex = self.NetworkIndex + devDesc.MemoryLimit = self.DeviceMemorySize + devDesc.SmUtilLimit = self.SmUtilLimit + devDesc.GpuType = self.GpuType + return devDesc +} + +func (self *SGuestIsolatedDevice) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { + devDesc := self.getDesc() + desc := jsonutils.NewDict() + desc.Update(jsonutils.Marshal(devDesc)) + desc.Add(jsonutils.NewString(GuestIsolatedDeviceManager.Keyword()), "res_name") + return desc +} diff --git a/pkg/compute/models/guest_queries.go b/pkg/compute/models/guest_queries.go index f40310669d..ae2b723cf0 100644 --- a/pkg/compute/models/guest_queries.go +++ b/pkg/compute/models/guest_queries.go @@ -952,8 +952,8 @@ func fetchContainers(guestIds []string) (map[string][]*api.PodContainerDesc, err } func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevice { - q := IsolatedDeviceManager.Query().In("guest_id", guestIds) - devs := make([]SIsolatedDevice, 0) + q := GuestIsolatedDeviceManager.Query().In("guest_id", guestIds) + devs := make([]SGuestIsolatedDevice, 0) err := q.All(&devs) if err != nil { return nil @@ -961,14 +961,14 @@ func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevi ret := make(map[string][]api.SIsolatedDevice) for i := range devs { dev := api.SIsolatedDevice{} - dev.Id = devs[i].Id - dev.HostId = devs[i].HostId - dev.DevType = devs[i].DevType - dev.Model = devs[i].Model - dev.GuestId = devs[i].GuestId - dev.Addr = devs[i].Addr - dev.VendorDeviceId = devs[i].VendorDeviceId - dev.NumaNode = byte(devs[i].NumaNode) + gdev := devs[i].GetIsolatedDevice() + dev.Id = gdev.Id + dev.HostId = gdev.HostId + dev.DevType = gdev.DevType + dev.Model = gdev.Model + dev.Addr = gdev.Addr + dev.VendorDeviceId = gdev.VendorDeviceId + dev.NumaNode = byte(gdev.NumaNode) gdevs, ok := ret[devs[i].GuestId] if !ok { gdevs = make([]api.SIsolatedDevice, 0) diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 3a0cb1ea38..9417e22a7c 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -258,8 +258,8 @@ type IGuestDriver interface { ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error - BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error - BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error + BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SGuestIsolatedDevice) error + BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SGuestIsolatedDevice) error RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index a153187411..7acc02969f 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -1010,7 +1010,7 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, userCr return errors.Wrapf(httperrors.ErrInvalidStatus, "eip associate with %s", gn.IpAddr) } guest := gn.GetGuest() - dev, err := guest.GetIsolatedDeviceByNetworkIndex(gn.Index) + dev, err := guest.GetGuestIsolatedDeviceByNetworkIndex(gn.Index) if err != nil { return errors.Wrap(err, "GetIsolatedDeviceByNetworkIndex") } @@ -1274,7 +1274,10 @@ func (gn *SGuestnetwork) GetVirtualIPs() []string { func (gn *SGuestnetwork) GetIsolatedDevice() (*SIsolatedDevice, error) { dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", gn.GuestId).Equals("network_index", gn.Index) + q := IsolatedDeviceManager.Query() + gidq := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", gn.GuestId).Equals("network_index", gn.Index).SubQuery() + q = q.Join(gidq, sqlchemy.Equals(q.Field("id"), gidq.Field("isolated_device_id"))) if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 248048b70c..b5b5b39e35 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -561,12 +561,14 @@ func (manager *SGuestManager) ListItemFilter( devTypeQ := func(q *sqlchemy.SQuery, checkType, backup *bool, dType string, conditions []sqlchemy.ICondition) []sqlchemy.ICondition { if checkType != nil { - isodev := IsolatedDeviceManager.Query().SubQuery() - isodevCons := []sqlchemy.ICondition{sqlchemy.IsNotNull(isodev.Field("guest_id"))} + guestIdev := GuestIsolatedDeviceManager.Query().SubQuery() + sgq := guestIdev.Query(guestIdev.Field("guest_id")).GroupBy(guestIdev.Field("guest_id")) + if len(dType) > 0 { - isodevCons = append(isodevCons, sqlchemy.Startswith(isodev.Field("dev_type"), dType)) + isodev := IsolatedDeviceManager.Query().SubQuery() + sgq = sgq.Join(isodev, sqlchemy.Equals(guestIdev.Field("isolated_device_id"), isodev.Field("id"))) + sgq = sgq.Filter(sqlchemy.Startswith(isodev.Field("dev_type"), dType)) } - sgq := isodev.Query(isodev.Field("guest_id")).Filter(sqlchemy.AND(isodevCons...)) cond := sqlchemy.NotIn if *checkType { cond = sqlchemy.In @@ -729,6 +731,12 @@ func (manager *SGuestManager) ListItemFilter( q = q.IsNullOrEmpty("host_id") } } + if len(query.IsolatedDeviceId) > 0 { + sq := GuestIsolatedDeviceManager.Query("guest_id"). + Equals("isolated_device_id", query.IsolatedDeviceId).SubQuery() + q = q.In("id", sq) + } + if len(query.SnapshotpolicyId) > 0 { sp := SnapshotPolicyResourceManager.Query("resource_id"). Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER). @@ -2008,7 +2016,7 @@ func (manager *SGuestManager) validateCreateData( hasGpuVga := func() bool { for i := 0; i < len(input.IsolatedDevices); i++ { - if input.IsolatedDevices[i].DevType == api.GPU_VGA_TYPE { + if input.IsolatedDevices[i].GpuType == api.GPU_VGA { return true } } @@ -2314,9 +2322,9 @@ func (manager *SGuestManager) validateCreateData( nvidiaVgpuCnt := 0 gpuCnt := 0 for i := 0; i < len(input.IsolatedDevices); i++ { - if input.IsolatedDevices[i].DevType == api.LEGACY_VGPU_TYPE { + if input.IsolatedDevices[i].SharingMode == api.DEVICE_SHARING_MODE_MDEV { nvidiaVgpuCnt += 1 - } else if utils.IsInStringArray(input.IsolatedDevices[i].DevType, api.VALID_GPU_TYPES) { + } else if input.IsolatedDevices[i].DevType == api.GPU_TYPE { gpuCnt += 1 } } @@ -4741,11 +4749,10 @@ func _guestResourceCountQuery( diskBackupSubQuery := backupDiskQuery.SubQuery() // diskBackupSubQuery := diskQuery.IsNotEmpty("backup_storage_id").SubQuery() - isolated := IsolatedDeviceManager.Query().SubQuery() + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() - isoDevQuery := isolated.Query(isolated.Field("guest_id"), sqlchemy.COUNT("device_sum")) - isoDevQuery = isoDevQuery.Filter(sqlchemy.IsNotNull(isolated.Field("guest_id"))) - isoDevQuery = isoDevQuery.GroupBy(isolated.Field("guest_id")) + isoDevQuery := guestIdevs.Query(guestIdevs.Field("guest_id"), sqlchemy.COUNT("device_sum")) + isoDevQuery = isoDevQuery.GroupBy(guestIdevs.Field("guest_id")) isoDevSubQuery := isoDevQuery.SubQuery() @@ -4897,10 +4904,11 @@ func (self *SGuest) allocSriovNicDevice( if err != nil { return errors.Wrap(err, "self.createIsolatedDeviceOnHost") } - dev, err := self.GetIsolatedDeviceByNetworkIndex(gn.Index) + gdev, err := self.GetGuestIsolatedDeviceByNetworkIndex(gn.Index) if err != nil { return errors.Wrap(err, "self.GetIsolatedDeviceByNetworkIndex") } + dev := gdev.GetIsolatedDevice() if dev.OvsOffloadInterface != "" { _, err = db.Update(gn, func() error { gn.Ifname = dev.OvsOffloadInterface @@ -4977,9 +4985,9 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie dev, _ := idev.(*SIsolatedDevice) sriovWires = []string{dev.WireId} } else { - wires, err := IsolatedDeviceManager.FindUnusedNicWiresByModel(netConfig.SriovDevice.Model) + wires, err := IsolatedDeviceManager.FindAvailableNicWiresByModel(netConfig.SriovDevice.Model) if err != nil { - return nil, errors.Wrap(err, "FindUnusedNicWiresByModel") + return nil, errors.Wrap(err, "FindAvailableNicWiresByModel") } sriovWires = wires } @@ -5719,7 +5727,9 @@ func (self *SGuest) getExtraOptions(ctx context.Context) jsonutils.JSONObject { } func (self *SGuest) GetIsolatedDevices() ([]SIsolatedDevice, error) { - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id) + gq := GuestIsolatedDeviceManager.Query().Equals("guest_id", self.Id).SubQuery() + q := IsolatedDeviceManager.Query() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) devs := []SIsolatedDevice{} err := db.FetchModelObjects(IsolatedDeviceManager, q, &devs) if err != nil { @@ -5728,9 +5738,10 @@ func (self *SGuest) GetIsolatedDevices() ([]SIsolatedDevice, error) { return devs, nil } -func (self *SGuest) GetIsolatedDeviceByNetworkIndex(index int) (*SIsolatedDevice, error) { - dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id).Equals("network_index", index) +func (self *SGuest) GetGuestIsolatedDeviceByNetworkIndex(index int) (*SGuestIsolatedDevice, error) { + dev := SGuestIsolatedDevice{} + q := GuestIsolatedDeviceManager.Query().Equals("network_index", index).Equals("guest_id", self.Id) + if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { @@ -5740,13 +5751,17 @@ func (self *SGuest) GetIsolatedDeviceByNetworkIndex(index int) (*SIsolatedDevice if err != nil { return nil, err } - dev.SetModelManager(IsolatedDeviceManager, &dev) + dev.SetModelManager(GuestIsolatedDeviceManager, &dev) return &dev, nil } func (self *SGuest) GetIsolatedDeviceByDiskIndex(index int8) (*SIsolatedDevice, error) { dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id).Equals("disk_index", index) + q := IsolatedDeviceManager.Query() + gidq := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", self.Id).Equals("disk_index", index).SubQuery() + q = q.Join(gidq, sqlchemy.Equals(q.Field("id"), gidq.Field("isolated_device_id"))) + if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { @@ -5804,8 +5819,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *a desc.IsVolatileHost = true } - // isolated devices - isolatedDevs, _ := self.GetIsolatedDevices() + isolatedDevs, _ := self.GetGuestIsolatedDevices() for _, dev := range isolatedDevs { desc.IsolatedDevices = append(desc.IsolatedDevices, dev.getDesc()) } @@ -6065,10 +6079,11 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { spec.Set("nic", nicSpecs) // get isolate device spec - guestgpus, _ := self.GetIsolatedDevices() + guestgpus, _ := self.GetGuestIsolatedDevices() gpuSpecs := []GpuSpec{} - for _, guestgpu := range guestgpus { - if strings.HasPrefix(guestgpu.DevType, "GPU") { + for i := range guestgpus { + guestgpu := guestgpus[i].GetIsolatedDevice() + if guestgpu.DevType == api.GPU_TYPE { gs := guestgpu.GetGpuSpec() gpuSpecs = append(gpuSpecs, *gs) } @@ -7198,16 +7213,17 @@ func (self *SGuest) ToNetworksConfig() []*api.NetworkConfig { } func (self *SGuest) ToIsolatedDevicesConfig() []*api.IsolatedDeviceConfig { - guestIsolatedDevices, _ := self.GetIsolatedDevices() + guestIsolatedDevices, _ := self.GetGuestIsolatedDevices() if len(guestIsolatedDevices) == 0 { return nil } ret := make([]*api.IsolatedDeviceConfig, len(guestIsolatedDevices)) - for idx, guestIsolatedDevice := range guestIsolatedDevices { + for idx := range guestIsolatedDevices { + dev := guestIsolatedDevices[idx].GetIsolatedDevice() devConf := new(api.IsolatedDeviceConfig) - devConf.Model = guestIsolatedDevice.Model - devConf.Vendor = guestIsolatedDevice.getVendor() - devConf.DevType = guestIsolatedDevice.DevType + devConf.Model = dev.Model + devConf.Vendor = dev.getVendor() + devConf.DevType = dev.DevType ret[idx] = devConf } return ret diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a9dc3329d9..f4247ffdfb 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3526,13 +3526,13 @@ func (manager *SHostManager) totalCountQ( q = db.ObjectIdQueryWithPolicyResult(ctx, q, HostManager, policyResult) - isolatedDevices := IsolatedDeviceManager.Query().SubQuery() + isolatedDevices := IsolatedDeviceManager.queryWithoutGuest(IsolatedDeviceManager.Query()).SubQuery() iq := isolatedDevices.Query( isolatedDevices.Field("host_id"), sqlchemy.SUM("isolated_reserved_memory", isolatedDevices.Field("reserved_memory")), sqlchemy.SUM("isolated_reserved_cpu", isolatedDevices.Field("reserved_cpu")), sqlchemy.SUM("isolated_reserved_storage", isolatedDevices.Field("reserved_storage")), - ).IsNullOrEmpty("guest_id").GroupBy(isolatedDevices.Field("host_id")).SubQuery() + ).GroupBy(isolatedDevices.Field("host_id")).SubQuery() q = q.LeftJoin(iq, sqlchemy.Equals(q.Field("id"), iq.Field("host_id"))) q.AppendField( iq.Field("isolated_reserved_memory"), @@ -4110,7 +4110,25 @@ func (hh *SHost) GetDevsReservedResource(devs []SIsolatedDevice) *api.IsolatedDe ReservedCpu: &reservedCpu, } for _, dev := range devs { - if !utils.IsInStringArray(dev.DevType, api.VALID_GPU_TYPES) { + if !dev.IsKvmExclusiveGPU() { + continue + } + reservedCpu += dev.ReservedCpu + reservedMem += dev.ReservedMemory + reservedStorage += dev.ReservedStorage + } + return &reservedResourceForGpu +} + +func (hh *SHost) GetDevsReservedResourceByDevStats(devs []IsolatedDeviceAllocateStat) *api.IsolatedDeviceReservedResourceInput { + reservedCpu, reservedMem, reservedStorage := 0, 0, 0 + reservedResourceForGpu := api.IsolatedDeviceReservedResourceInput{ + ReservedStorage: &reservedStorage, + ReservedMemory: &reservedMem, + ReservedCpu: &reservedCpu, + } + for _, dev := range devs { + if !dev.IsKvmExclusiveGPU() { continue } reservedCpu += dev.ReservedCpu @@ -4422,6 +4440,19 @@ func (hh *SHost) GetDetailsIpmi(ctx context.Context, userCred mcclient.TokenCred return ret, nil } +func (hh *SHost) GetDetailsGuestIsolatedDevicesInitialized(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + inited, err := IsolatedDeviceManager.isInitializeDataDone() + if err != nil { + return nil, err + } + if !inited { + return nil, httperrors.NewResourceNotReadyError("isolated device not isitialized") + } + ret := jsonutils.NewDict() + ret.Set("initialized", jsonutils.NewString("ok")) + return ret, nil +} + func (manager *SHostManager) GetHostsByManagerAndRegion(managerId string, regionId string) []SHost { zones := ZoneManager.Query().Equals("cloudregion_id", regionId).SubQuery() hosts := HostManager.Query() @@ -8202,10 +8233,10 @@ func (h *SHost) GetDetailsApiStats(ctx context.Context, userCred mcclient.TokenC } func (hh *SHost) GetDetailsIsolatedDeviceNumaStats(ctx context.Context, userCred mcclient.TokenCredential, input *api.HostIsolatedDeviceNumaStatsInput) (jsonutils.JSONObject, error) { - if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { - return nil, httperrors.NewInputParameterError("dev_type %s is invalid", input.DevType) + if input.Model == "" { + return nil, httperrors.NewMissingParameterError("model") } - stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.DevType, hh.Id) + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.Model, hh.Id) if err != nil { return nil, err } diff --git a/pkg/compute/models/initdb.go b/pkg/compute/models/initdb.go index b706ad3cbc..98d6d52394 100644 --- a/pkg/compute/models/initdb.go +++ b/pkg/compute/models/initdb.go @@ -51,6 +51,8 @@ func InitDB() error { GuestManager, GuestnetworkManager, HostManager, + GuestIsolatedDeviceManager, + IsolatedDeviceManager, HostDmesgLogManager, LoadbalancerCertificateManager, LoadbalancerAclManager, diff --git a/pkg/compute/models/isolated_device_models.go b/pkg/compute/models/isolated_device_models.go index ee6b79555c..1360721eaf 100644 --- a/pkg/compute/models/isolated_device_models.go +++ b/pkg/compute/models/isolated_device_models.go @@ -24,7 +24,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -77,10 +76,6 @@ func (manager *SIsolatedDeviceModelManager) ValidateCreateData(ctx context.Conte query jsonutils.JSONObject, input api.IsolatedDeviceModelCreateInput, ) (api.IsolatedDeviceModelCreateInput, error) { - if utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { - return input, httperrors.NewInputParameterError("device type %q is not supported", input.DevType) - } - input.VendorId = strings.ToLower(input.VendorId) input.DeviceId = strings.ToLower(input.DeviceId) deviceVendorReg := regexp.MustCompile(`^[a-f0-9]{4}$`) diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 1d22102d22..e4b66282d0 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -31,7 +31,6 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/rbacscope" - "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -48,25 +47,6 @@ import ( "yunion.io/x/onecloud/pkg/util/stringutils2" ) -const ( -//DIRECT_PCI_TYPE = api.DIRECT_PCI_TYPE -//GPU_HPC_TYPE = api.GPU_HPC_TYPE // # for compute -//GPU_VGA_TYPE = api.GPU_VGA_TYPE // # for display -//USB_TYPE = api.USB_TYPE -//NIC_TYPE = api.NIC_TYPE - -// NVIDIA_VENDOR_ID = api.NVIDIA_VENDOR_ID -// AMD_VENDOR_ID = api.AMD_VENDOR_ID -) - -var VALID_GPU_TYPES = api.VALID_GPU_TYPES - -var VALID_PASSTHROUGH_TYPES = api.VALID_PASSTHROUGH_TYPES - -var ID_VENDOR_MAP = api.ID_VENDOR_MAP - -var VENDOR_ID_MAP = api.VENDOR_ID_MAP - type SIsolatedDeviceManager struct { db.SStandaloneResourceBaseManager db.SExternalizedResourceBaseManager @@ -76,6 +56,12 @@ type SIsolatedDeviceManager struct { var IsolatedDeviceManager *SIsolatedDeviceManager +const ( + isolatedDeviceInitializeDataObjType = "system" + isolatedDeviceInitializeDataObjId = "compute_isolated_device_initialize_data" + isolatedDeviceInitializeDataKey = "__initialized" +) + func init() { gotypes.RegisterSerializable(reflect.TypeOf(&api.IsolatedDevicePCIEInfo{}), func() gotypes.ISerializable { return &api.IsolatedDevicePCIEInfo{} @@ -98,17 +84,16 @@ type SIsolatedDevice struct { db.SSharableBaseResource `"is_public->create":"domain_optional" "public_scope->create":"domain_optional"` SHostResourceBase `width:"36" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required"` - // # PCI / GPU-HPC / GPU-VGA / USB / NIC + // # PCI / GPU / USB / NIC ... // 设备类型 DevType string `width:"128" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"` - + // EXCLUSIVE / SRIOV / MPS / HAMI / SHARE / MIG + SharingMode string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_required"` + // Device is hot pluggable + HotPluggable bool `default:"false" list:"domain" create:"domain_optional" update:"domain"` // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `width:"512" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"` - // 云主机Id - GuestId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain"` - // guest network index - NetworkIndex int `nullable:"true" default:"-1" list:"user" update:"user"` // Nic wire id WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"` // Offload interface name @@ -117,11 +102,6 @@ type SIsolatedDevice struct { IsInfinibandNic bool `nullable:"false" default:"false" list:"user" create:"optional"` // NVME disk size NvmeSizeMB int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"` - // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, - // or per-slice quota for MPS share mode). 0 means unknown / not applicable. - MemorySize int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` - // guest disk index - DiskIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"` // # pci address of `Bus:Device.Function` format, or usb bus address of `bus:addr:port` Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` @@ -177,6 +157,20 @@ type SIsolatedDevice struct { PcieInfo *api.IsolatedDevicePCIEInfo `nullable:"true" create:"optional" list:"user" get:"user" update:"domain"` // device numa node NumaNode int8 `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"` + + // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, + // or per-slice quota for MPS share mode). 0 means unknown / not applicable. + MemorySize int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` + // some of isolated device type support virtual num, like NVIDIA_GPU_SHARE, NVIDIA_MPS + VirtualNum int `nullable:"true" default:"1" list:"user" update:"domain" create:"domain_optional"` +} + +func (manager *SIsolatedDeviceManager) GetIVirtualModelManager() db.IVirtualModelManager { + return manager +} + +func (manager *SIsolatedDeviceManager) GetResourceCount() ([]db.SScopeResourceCount, error) { + return []db.SScopeResourceCount{}, nil } func (manager *SIsolatedDeviceManager) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition { @@ -204,12 +198,16 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, if input.DevType == "" { return input, httperrors.NewNotEmptyError("dev_type is empty") } - if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { + if !utils.IsInStringArray(input.DevType, api.VALID_TYPES) { if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil { return input, httperrors.NewInputParameterError("device type %q is not supported", input.DevType) } } + if !utils.IsInStringArray(input.SharingMode, api.VAILD_SHARING_MODES) { + return input, httperrors.NewNotEmptyError("sharing_mode %s is not valid", input.SharingMode) + } + input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput) if err != nil { return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData") @@ -225,7 +223,7 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, // validate reserverd resource // inject default reserverd resource for gpu: - if utils.IsInStringArray(input.DevType, []string{api.GPU_HPC_TYPE, api.GPU_VGA_TYPE}) { + if host.HostType == api.HOST_TYPE_KVM && input.SharingMode == api.DEVICE_SHARING_MODE_EXCLUSIVE && input.DevType == api.GPU_TYPE { defaultCPU := 8 // 8 defaultMem := 8192 // 8g defaultStore := 102400 // 100g @@ -270,18 +268,6 @@ func (self *SIsolatedDevice) ValidateUpdateData( if input.ReservedStorage != nil && *input.ReservedStorage < 0 { return input, httperrors.NewInputParameterError("reserved storage must >= 0") } - if input.DevType != "" && input.DevType != self.DevType { - if !utils.IsInStringArray(input.DevType, api.VALID_GPU_TYPES) { - if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil { - return input, httperrors.NewInputParameterError("device type %q does not support update", input.DevType) - } - } else { - if !self.IsGPU() { - return input, httperrors.NewInputParameterError("Can't update for device %q", self.DevType) - } - } - } - return input, nil } @@ -342,7 +328,7 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( q = q.Equals("dev_type", "USB") } if query.Unused != nil && *query.Unused { - q = q.IsEmpty("guest_id") + q = manager.queryWithoutGuest(q) } if len(query.DevType) > 0 { @@ -380,7 +366,8 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( if err != nil { return nil, errors.Wrapf(err, "Fetch guest by %q", query.GuestId) } - q = q.Equals("guest_id", obj.GetId()) + gq := GuestIsolatedDeviceManager.Query().Equals("guest_id", obj.GetId()).SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) } return q, nil @@ -425,7 +412,9 @@ func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q } if keys.Contains("guest") { guestNameQuery := GuestManager.Query("name", "id").SubQuery() - q.LeftJoin(guestNameQuery, sqlchemy.Equals(q.Field("guest_id"), guestNameQuery.Field("id"))) + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + q.LeftJoin(guestNameQuery, sqlchemy.Equals(gq.Field("guest_id"), guestNameQuery.Field("id"))) q.AppendField(guestNameQuery.Field("name", "guest")) } if keys.Contains("host") { @@ -448,7 +437,11 @@ func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, k } func (self *SIsolatedDevice) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { - if len(self.GuestId) > 0 { + gdevs, err := self.GetAllGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) > 0 { return httperrors.NewNotEmptyError("Isolated device used by server") } return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx, nil) @@ -458,18 +451,7 @@ func (self *SIsolatedDevice) getDetailedString() string { return fmt.Sprintf("%s:%s/%s/%s", self.Addr, self.Model, self.VendorDeviceId, self.DevType) } -func (manager *SIsolatedDeviceManager) findAttachedDevicesOfGuest(guest *SGuest) []SIsolatedDevice { - devs := make([]SIsolatedDevice, 0) - q := manager.Query().Equals("guest_id", guest.Id) - err := db.FetchModelObjects(manager, q, &devs) - if err != nil { - log.Errorf("findAttachedDevicesOfGuest error %s", err) - return nil - } - return devs -} - -func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr string, devType string) *SIsolatedDevice { +func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr, devType, sharingMode string) *SIsolatedDevice { dev := SIsolatedDevice{} dev.SetModelManager(manager, &dev) @@ -477,6 +459,9 @@ func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr string, devType if devType != "" { q = q.Equals("dev_type", devType) } + if sharingMode != "" { + q = q.Equals("sharing_mode", sharingMode) + } if fuzzyStr != "" { qe := q.Equals("model", fuzzyStr) @@ -501,7 +486,7 @@ func (self *SIsolatedDevice) getVendorId() string { func (self *SIsolatedDevice) getVendor() string { vendorId := self.getVendorId() - vendor, ok := ID_VENDOR_MAP[vendorId] + vendor, ok := api.ID_VENDOR_MAP[vendorId] if ok { return vendor } else { @@ -512,7 +497,7 @@ func (self *SIsolatedDevice) getVendor() string { func GetVendorByVendorDeviceId(vendorDeviceId string) string { parts := strings.Split(vendorDeviceId, ":") vendorId := parts[0] - vendor, ok := ID_VENDOR_MAP[vendorId] + vendor, ok := api.ID_VENDOR_MAP[vendorId] if ok { return vendor } else { @@ -521,7 +506,7 @@ func GetVendorByVendorDeviceId(vendorDeviceId string) string { } func (self *SIsolatedDevice) IsGPU() bool { - return strings.HasPrefix(self.DevType, "GPU") || sets.NewString(api.CONTAINER_GPU_TYPES...).Has(self.DevType) + return self.DevType == api.GPU_TYPE } func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) { @@ -529,7 +514,7 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr var matchDev *SIsolatedDevice devId = devConfig.Id - matchDev = manager.fuzzyMatchModel(devConfig.Model, devConfig.DevType) + matchDev = manager.fuzzyMatchModel(devConfig.Model, devConfig.DevType, devConfig.SharingMode) devVendor = devConfig.Vendor devType = devConfig.DevType @@ -538,8 +523,10 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr return nil, httperrors.NewNotFoundError("Not found matched device by model: %q, dev_type: %q", devConfig.Model, devConfig.DevType) } devConfig.Model = matchDev.Model + devConfig.SharingMode = matchDev.SharingMode + if len(devVendor) > 0 { - vendorId, ok := VENDOR_ID_MAP[devVendor] + vendorId, ok := api.VENDOR_ID_MAP[devVendor] if ok { devConfig.Vendor = vendorId } else { @@ -558,17 +545,21 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr devConfig.Id = dev.Id devConfig.Model = dev.Model devConfig.DevType = dev.DevType + devConfig.SharingMode = dev.SharingMode devConfig.Vendor = dev.getVendor() devConfig.WireId = dev.WireId - if dev.IsGPU() && len(devType) > 0 { - if !utils.IsInStringArray(devType, VALID_GPU_TYPES) { - return nil, fmt.Errorf("%s not valid for GPU device", devType) - } + if devType != "" && devType != dev.DevType { + return nil, fmt.Errorf("request dev_type %s not match dev %s type %s", devType, dev.Id, dev.DevType) } } if len(devType) > 0 { devConfig.DevType = devType } + if devConfig.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if devConfig.MemoryRequest <= 0 { + return nil, httperrors.NewBadRequestError("dev sharing_mode %s must give memory request", devConfig.SharingMode) + } + } return devConfig, nil } @@ -579,10 +570,13 @@ func (manager *SIsolatedDeviceManager) isValidDeviceInfo(config *api.IsolatedDev return httperrors.NewResourceNotFoundError("IsolatedDevice %s not found", config.Id) } dev := devObj.(*SIsolatedDevice) - if len(dev.GuestId) > 0 { - return httperrors.NewConflictError("Isolated device already attached to another guest: %s", dev.GuestId) + if dev.IsFull() { + return httperrors.NewConflictError("Isolated device already attached") } } + if config.GpuType != "" && !utils.IsInStringArray(config.GpuType, []string{api.GPU_HPC, api.GPU_VGA}) { + return httperrors.NewInputParameterError("Input gpu_type %s not valid", config.GpuType) + } return nil } @@ -601,8 +595,8 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe return httperrors.NewResourceNotFoundError("IsolatedDevice %s not found", config.Id) } dev := devObj.(*SIsolatedDevice) - if len(dev.GuestId) > 0 { - return httperrors.NewConflictError("Isolated device already attached to another guest: %s", dev.GuestId) + if dev.IsFull() { + return httperrors.NewConflictError("Isolated device already attached") } if dev.DevType != devType { return httperrors.NewBadRequestError("IsolatedDevice is not device type %s", devType) @@ -635,7 +629,10 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C if len(devConfig.DevType) > 0 && devConfig.DevType != dev.DevType { dev.DevType = devConfig.DevType } - return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex) + if !dev.IsEnough(devConfig.MemoryRequest) { + return errors.Errorf("Dev %s not enough", dev.Id) + } + return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int) error { @@ -643,7 +640,7 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig) } // if dev type is not nic, wire is empty string - devs, err := manager.findHostUnusedByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId) + devs, err := manager.findHostAvailableByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId) if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) } @@ -652,8 +649,13 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c return fmt.Errorf("device_path %s on host %s does not satisfy memory_mb=%d", devConfig.DevicePath, host.Id, devConfig.MemoryMb) } + var selectedDev SIsolatedDevice for i := range devs { + if !devs[i].IsEnough(devConfig.MemoryRequest) { + continue + } + if _, ok := usedDevMap[devs[i].DevicePath]; !ok { selectedDev = devs[i] usedDevMap[devs[i].DevicePath] = &selectedDev @@ -662,7 +664,10 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c if selectedDev.Id == "" { selectedDev = devs[0] } - return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) + if !selectedDev.IsEnough(devConfig.MemoryRequest) { + return errors.Errorf("Dev %s not enough", selectedDev.Id) + } + return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } // filterDevicesByMemoryMb drops devices whose MemorySize > 0 and is below the @@ -836,7 +841,7 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( return fmt.Errorf("Not found model from info: %#v", devConfig) } // if dev type is not nic, wire is empty string - devs, err := manager.findHostUnusedByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId) + devs, err := manager.findHostAvailableByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId) if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s on host %s", devConfig.Model, host.Id) } @@ -852,6 +857,10 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( //groupDevs := make(SorttedGroupDevs, 0) mapDevs := map[string][]SIsolatedDevice{} for i := range devs { + if !devs[i].IsEnough(devConfig.MemoryRequest) { + continue + } + dev := devs[i] devPath := dev.DevicePath var gdevs []SIsolatedDevice @@ -1048,27 +1057,12 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( devAddr := strings.Split(selectedDev.Addr, "-")[0] usedDevMap[devAddr] = selectedDev - return guest.attachIsolatedDevice(ctx, userCred, selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) + return guest.attachIsolatedDevice(ctx, userCred, selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } -func (manager *SIsolatedDeviceManager) findUnusedQuery() *sqlchemy.SQuery { - isolateddevs := manager.Query().SubQuery() - q := isolateddevs.Query().Filter(sqlchemy.OR(sqlchemy.IsNull(isolateddevs.Field("guest_id")), - sqlchemy.IsEmpty(isolateddevs.Field("guest_id")))) - return q -} - -func (manager *SIsolatedDeviceManager) UnusedGpuQuery() *sqlchemy.SQuery { - q := manager.findUnusedQuery() - q = q.Filter(sqlchemy.OR( - sqlchemy.Equals(q.Field("dev_type"), api.GPU_HPC_TYPE), - sqlchemy.Equals(q.Field("dev_type"), api.GPU_VGA_TYPE))) - return q -} - -func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) FindAvailableByModels(models []string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.findUnusedQuery() + q := manager.GetAvailableIsolatedDeviceQuery(nil) q = q.In("model", models) err := db.FetchModelObjects(manager, q, &devs) if err != nil { @@ -1077,8 +1071,8 @@ func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SI return devs, nil } -func (manager *SIsolatedDeviceManager) FindUnusedNicWiresByModel(modelName string) ([]string, error) { - q := manager.Query().IsNullOrEmpty("guest_id").Equals("dev_type", api.NIC_TYPE) +func (manager *SIsolatedDeviceManager) FindAvailableNicWiresByModel(modelName string) ([]string, error) { + q := manager.Query().Equals("dev_type", api.NIC_TYPE) if len(modelName) > 0 { q = q.Equals("model", modelName) } @@ -1090,15 +1084,19 @@ func (manager *SIsolatedDeviceManager) FindUnusedNicWiresByModel(modelName strin } wires := make([]string, len(devs)) for i := 0; i < len(devs); i++ { + if devs[i].IsFull() { + continue + } + wires[i] = devs[i].WireId } return wires, err } -func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) FindAvailableGpusOnHost(hostId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.UnusedGpuQuery() - q = q.Equals("host_id", hostId) + q := manager.GetAvailableIsolatedDeviceQuery(nil) + q = q.Equals("dev_type", api.GPU_TYPE).Equals("host_id", hostId) err := db.FetchModelObjects(manager, q, &devs) if err != nil { return nil, err @@ -1106,8 +1104,8 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI return devs, nil } -func (manager *SIsolatedDeviceManager) findHostUnusedByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { - return manager.findHostUnusedByDevAttr(model, "dev_type", devType, hostId, wireId) +func (manager *SIsolatedDeviceManager) findHostAvailableByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { + return manager.findHostAvailableByDevAttr(model, "dev_type", devType, hostId, wireId) } func (manager *SIsolatedDeviceManager) findHostDevsByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { @@ -1133,9 +1131,9 @@ func (manager *SIsolatedDeviceManager) findHostDevsByDevAttr(model, attrKey, att return devs, nil } -func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) findHostAvailableByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.findUnusedQuery() + q := manager.GetAvailableIsolatedDeviceQuery(nil) q = q.Equals("model", model).Equals("host_id", hostId) if attrVal != "" { q.Equals(attrKey, attrVal) @@ -1154,20 +1152,19 @@ func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, a } func (manager *SIsolatedDeviceManager) ReleaseGPUDevicesOfGuest(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential) error { - devs := manager.findAttachedDevicesOfGuest(guest) - if devs == nil { + gdevs, err := guest.GetGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) == 0 { return fmt.Errorf("fail to find attached devices") } - for _, dev := range devs { - if !utils.IsInStringArray(dev.DevType, api.VALID_GPU_TYPES) { + for _, gdev := range gdevs { + dev := gdev.GetIsolatedDevice() + if !dev.IsKvmExclusiveGPU() { continue } - _, err := db.Update(&dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - dev.DiskIndex = -1 - return nil - }) + err := gdev.Detach(ctx, userCred) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_GUEST_DETACH_ISOLATED_DEVICE_FAIL, dev.GetShortDesc(ctx), userCred) return err @@ -1178,16 +1175,16 @@ func (manager *SIsolatedDeviceManager) ReleaseGPUDevicesOfGuest(ctx context.Cont } func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential) error { - devs := manager.findAttachedDevicesOfGuest(guest) - if devs == nil { + gdevs, err := guest.GetGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) == 0 { return fmt.Errorf("fail to find attached devices") } - for _, dev := range devs { - _, err := db.Update(&dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - return nil - }) + for _, gdev := range gdevs { + dev := gdev.GetIsolatedDevice() + err := gdev.Detach(ctx, userCred) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_GUEST_DETACH_ISOLATED_DEVICE_FAIL, dev.GetShortDesc(ctx), userCred) return err @@ -1197,6 +1194,19 @@ func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context return nil } +func (manager *SIsolatedDeviceManager) queryWithoutGuest(q *sqlchemy.SQuery) *sqlchemy.SQuery { + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.LeftJoin(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + q = q.Filter(sqlchemy.IsNull(gq.Field("isolated_device_id"))) + return q +} + +func (manager *SIsolatedDeviceManager) queryWithGuest(q *sqlchemy.SQuery) *sqlchemy.SQuery { + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + return q +} + func (manager *SIsolatedDeviceManager) totalCountQ( ctx context.Context, scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, devType []string, hostTypes []string, @@ -1260,12 +1270,14 @@ func (manager *SIsolatedDeviceManager) totalCount( policyResult, ) sq := iq.SubQuery() + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() q := sq.Query( sq.Field("dev_type"), - sq.Field("guest_id"), + guestIdevs.Field("guest_id"), sqlchemy.COUNT("count", sq.Field("id")), ) - q = q.GroupBy(q.Field("dev_type"), q.Field("guest_id")) + q = q.LeftJoin(guestIdevs, sqlchemy.Equals(sq.Field("id"), guestIdevs.Field("isolated_device_id"))) + q = q.GroupBy(sq.Field("dev_type"), guestIdevs.Field("guest_id")) ret := []IsolatedDeviceStat{} err := q.All(&ret) if err != nil { @@ -1297,12 +1309,12 @@ func (manager *SIsolatedDeviceManager) TotalCount( } for _, s := range stat { ret.Devices += s.Count - if utils.IsInStringArray(s.DevType, VALID_GPU_TYPES) { + if s.DevType == api.GPU_TYPE { ret.Gpus += s.Count } if len(s.GuestId) > 0 { ret.DevicesUsed += s.Count - if utils.IsInStringArray(s.DevType, VALID_GPU_TYPES) { + if s.DevType == api.GPU_TYPE { ret.GpusUsed += s.Count } } @@ -1315,12 +1327,11 @@ func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { Id: self.Id, DevType: self.DevType, Model: self.Model, + SharingMode: self.SharingMode, Addr: self.Addr, VendorDeviceId: self.VendorDeviceId, Vendor: self.getVendor(), - NetworkIndex: self.NetworkIndex, OvsOffloadInterface: self.OvsOffloadInterface, - DiskIndex: self.DiskIndex, NvmeSizeMB: self.NvmeSizeMB, MemorySize: self.MemorySize, MdevId: self.MdevId, @@ -1334,9 +1345,9 @@ func (man *SIsolatedDeviceManager) GetSpecShouldCheckStatus(query *jsonutils.JSO func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonutils.JSONObject, error) { hostQ := HostManager.Query() - q := man.Query("vendor_device_id", "model", "dev_type") + q := man.Query("vendor_device_id", "model", "dev_type", "sharing_mode", "nvme_size_mb", "memory_size") if statusCheck { - q = q.IsNullOrEmpty("guest_id") + q = man.GetAvailableIsolatedDeviceQuery(q) hostQ = hostQ.Equals("status", api.BAREMETAL_RUNNING).IsTrue("enabled"). In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER, api.HOST_TYPE_ZETTAKIT}) } @@ -1344,7 +1355,7 @@ func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonuti q.Join(hostSQ, sqlchemy.Equals(q.Field("host_id"), hostSQ.Field("id"))) q.AppendField(hostSQ.Field("host_type")) - q.GroupBy(hostSQ.Field("host_type"), q.Field("vendor_device_id"), q.Field("model"), q.Field("dev_type")) + q.GroupBy(hostSQ.Field("host_type"), q.Field("vendor_device_id"), q.Field("model"), q.Field("dev_type"), q.Field("sharing_mode"), q.Field("nvme_size_mb"), q.Field("memory_size")) q.AppendField(sqlchemy.COUNT("*")) rows, err := q.Rows() @@ -1355,46 +1366,54 @@ func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonuti res := jsonutils.NewDict() for rows.Next() { - var hostType, vendorDeviceId, m, t string + var hostType, vendorDeviceId, m, t, s string + var nvmeSize, memorySize int var count int - if err := rows.Scan(&vendorDeviceId, &m, &t, &hostType, &count); err != nil { + if err := rows.Scan(&vendorDeviceId, &m, &t, &s, &nvmeSize, &memorySize, &hostType, &count); err != nil { return nil, errors.Wrap(err, "get model spec scan rows") } vendor := GetVendorByVendorDeviceId(vendorDeviceId) - specKeys := man.getSpecKeys(vendor, m, t) + specKeys := man.getSpecKeys(vendor, m, t, s) specKey := GetSpecIdentKey(specKeys) - spec := man.getSpecByRows(hostType, vendorDeviceId, m, t, &count) + spec := man.getSpecByRows(hostType, vendorDeviceId, m, t, s, &nvmeSize, &memorySize, &count) res.Set(specKey, spec) } return res, nil } -func (man *SIsolatedDeviceManager) getSpecByRows(hostType, vendorDeviceId, model, devType string, count *int) *jsonutils.JSONDict { +func (man *SIsolatedDeviceManager) getSpecByRows(hostType, vendorDeviceId, model, devType, sharingMode string, nvmeSize, memorySize, count *int) *jsonutils.JSONDict { var vdev bool var hypervisor string - if utils.IsInStringArray(devType, api.VITRUAL_DEVICE_TYPES) { + if utils.IsInStringArray(sharingMode, api.VIRTUAL_SHARING_MODES) { vdev = true } - if utils.IsInStringArray(devType, api.VALID_CONTAINER_DEVICE_TYPES) { + switch hostType { + case api.HOST_TYPE_CONTAINER: hypervisor = api.HYPERVISOR_POD - } else { - hypervisor = api.HYPERVISOR_KVM - } - if hostType == api.HOST_TYPE_ZETTAKIT { + case api.HOST_TYPE_ZETTAKIT: hypervisor = api.HYPERVISOR_ZETTAKIT + default: + hypervisor = api.HYPERVISOR_KVM } ret := jsonutils.NewDict() ret.Set("virtual_dev", jsonutils.NewBool(vdev)) ret.Set("hypervisor", jsonutils.NewString(hypervisor)) ret.Set("dev_type", jsonutils.NewString(devType)) + ret.Set("sharing_mode", jsonutils.NewString(sharingMode)) ret.Set("model", jsonutils.NewString(model)) ret.Set("pci_id", jsonutils.NewString(vendorDeviceId)) ret.Set("vendor", jsonutils.NewString(GetVendorByVendorDeviceId(vendorDeviceId))) if count != nil { ret.Set("count", jsonutils.NewInt(int64(*count))) } + if nvmeSize != nil { + ret.Set("nvme_size_mb", jsonutils.NewInt(int64(*nvmeSize))) + } + if memorySize != nil { + ret.Set("memory_size_mb", jsonutils.NewInt(int64(*memorySize))) + } return ret } @@ -1410,7 +1429,8 @@ type GpuSpec struct { func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { host := self.getHost() if statusCheck { - if len(self.GuestId) > 0 { + gdevs, _ := self.GetAllGuestIsolatedDevices() + if len(gdevs) > 0 { return nil } if host.Status != api.BAREMETAL_RUNNING || !host.GetEnabled() || @@ -1418,7 +1438,7 @@ func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { return nil } } - return IsolatedDeviceManager.getSpecByRows(host.HostType, self.VendorDeviceId, self.Model, self.DevType, nil) + return IsolatedDeviceManager.getSpecByRows(host.HostType, self.VendorDeviceId, self.Model, self.DevType, self.SharingMode, &self.NvmeSizeMB, &self.MemorySize, nil) } func (self *SIsolatedDevice) GetGpuSpec() *GpuSpec { @@ -1435,14 +1455,16 @@ func (man *SIsolatedDeviceManager) GetSpecIdent(spec *jsonutils.JSONDict) []stri devType, _ := spec.GetString("dev_type") vendor, _ := spec.GetString("vendor") model, _ := spec.GetString("model") - return man.getSpecKeys(vendor, model, devType) + sharingMode, _ := spec.GetString("sharing_mode") + return man.getSpecKeys(vendor, model, devType, sharingMode) } -func (man *SIsolatedDeviceManager) getSpecKeys(vendor, model, devType string) []string { +func (man *SIsolatedDeviceManager) getSpecKeys(vendor, model, devType, sharingMode string) []string { keys := []string{ fmt.Sprintf("type:%s", devType), fmt.Sprintf("vendor:%s", vendor), fmt.Sprintf("model:%s", model), + fmt.Sprintf("sharing_mode:%s", sharingMode), } return keys } @@ -1458,13 +1480,6 @@ func (self *SIsolatedDevice) getHost() *SHost { return HostManager.FetchHostById(self.HostId) } -func (self *SIsolatedDevice) getGuest() *SGuest { - if len(self.GuestId) > 0 { - return GuestManager.FetchGuestById(self.GuestId) - } - return nil -} - func (manager *SIsolatedDeviceManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, @@ -1478,28 +1493,47 @@ func (manager *SIsolatedDeviceManager) FetchCustomizeColumns( stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) hostRows := manager.SHostResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) shareRows := manager.SSharableBaseResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - guestIds := make([]string, len(rows)) + guestIds := make([][]string, len(rows)) + guestIdsAll := make([]string, 0) for i := range rows { rows[i] = api.IsolateDeviceDetails{ StandaloneResourceDetails: stdRows[i], HostResourceInfo: hostRows[i], SharableResourceBaseInfo: shareRows[i], } - guestIds[i] = objs[i].(*SIsolatedDevice).GuestId + dev := objs[i].(*SIsolatedDevice) + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + rows[i].MemoryAllocated, _ = dev.getAllocatedMemorySize() + } else { + rows[i].AllocatedCount, _ = dev.getAllocatedCount() + } + guestIds[i] = dev.getAttachedGuestIds() + if len(guestIds[i]) > 0 { + guestIdsAll = append(guestIdsAll, guestIds[i]...) + } } guests := make(map[string]SGuest) - err := db.FetchStandaloneObjectsByIds(GuestManager, guestIds, &guests) + err := db.FetchStandaloneObjectsByIds(GuestManager, guestIdsAll, &guests) if err != nil { log.Errorf("db.FetchStandaloneObjectsByIds fail %s", err) return rows } for i := range rows { - if guest, ok := guests[guestIds[i]]; ok { - rows[i].Guest = guest.Name - rows[i].GuestStatus = guest.Status + nguests := guestIds[i] + if len(nguests) > 0 { + rows[i].Guest = make([]string, len(nguests)) + rows[i].GuestStatus = make([]string, len(nguests)) } + + for j := range nguests { + if guest, ok := guests[nguests[j]]; ok { + rows[i].Guest[j] = guest.Name + rows[i].GuestStatus[j] = guest.Status + } + } + } return rows @@ -1526,20 +1560,20 @@ func (self *SIsolatedDevice) PerformPurge(ctx context.Context, userCred mcclient } func (self *SIsolatedDevice) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - if len(self.GuestId) > 0 { + guestIsolatedDevices := self.getAttachedGuests() + if len(guestIsolatedDevices) > 0 { if !jsonutils.QueryBoolean(data, "purge", false) { - return httperrors.NewBadRequestError("%s: %s", api.ErrMsgIsolatedDeviceUsedByServer, self.GuestId) + return httperrors.NewBadRequestError("%s", api.ErrMsgIsolatedDeviceUsedByServer) } - iGuest, err := GuestManager.FetchById(self.GuestId) - if err != nil { - return err - } - guest := iGuest.(*SGuest) - err = guest.detachIsolateDevice(ctx, userCred, self) - if err != nil { - return err + + for i := range guestIsolatedDevices { + err := guestIsolatedDevices[i].Detach(ctx, userCred) + if err != nil { + return err + } } } + host := self.getHost() if host != nil { db.OpsLog.LogEvent(host, db.ACT_HOST_DETACH_ISOLATED_DEVICE, self.GetShortDesc(ctx), userCred) @@ -1581,12 +1615,9 @@ func (manager *SIsolatedDeviceManager) GetAllDevsOnHost(hostId string) ([]SIsola return devs, nil } -func (manager *SIsolatedDeviceManager) GetUnusedDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) GetAvailableIsolatedDeviceOnHost(hostId string, model, sharingMode string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.Query().Equals("host_id", hostId).Equals("model", model).IsNullOrEmpty("guest_id") - if count > 0 { - q = q.Limit(count) - } + q := manager.GetAvailableIsolatedDeviceQuery(manager.Query().Equals("host_id", hostId).Equals("model", model)).Equals("sharing_mode", sharingMode) err := db.FetchModelObjects(manager, q, &devs) if err != nil { return nil, err @@ -1607,10 +1638,10 @@ func (manager *SIsolatedDeviceManager) hostHasDevAddr(hostId, addr, mdevId strin } func (manager *SIsolatedDeviceManager) CheckModelIsEmpty(model, vendor, device, devType string) (bool, error) { - cnt, err := manager.Query().Equals("model", model). + cnt, err := manager.queryWithGuest(manager.Query().Equals("model", model). Equals("dev_type", devType). - Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device)). - IsNotEmpty("guest_id").CountWithError() + Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device))). + CountWithError() if err != nil { return false, err } @@ -1741,14 +1772,6 @@ func (model *SIsolatedDevice) syncWithCloudIsolateDevice(ctx context.Context, us return nil } -func (model *SIsolatedDevice) SetNetworkIndex(idx int) error { - _, err := db.Update(model, func() error { - model.NetworkIndex = idx - return nil - }) - return err -} - func (model *SIsolatedDevice) GetRequiredSharedDomainIds() []string { host := model.getHost() if host != nil { @@ -1786,12 +1809,15 @@ type HostIsolatedDevicesNumaStat struct { NumaNodeDevCount int } -func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devType, hostId string) ([]HostIsolatedDevicesNumaStat, error) { - q := manager.Query().Equals("host_id", hostId).Equals("dev_type", devType) - guestQ := GuestManager.Query().SubQuery() +func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devModel, hostId string) ([]HostIsolatedDevicesNumaStat, error) { + q := GuestIsolatedDeviceManager.Query() + guestQ := GuestManager.Query().NotEquals("status", api.VM_READY).SubQuery() + isq := manager.Query().Equals("host_id", hostId).Equals("model", devModel).SubQuery() + + q = q.Join(isq, sqlchemy.Equals(q.Field("isolated_device_id"), isq.Field("id"))) q = q.Join(guestQ, sqlchemy.Equals(q.Field("guest_id"), guestQ.Field("id"))) - q = q.Filter(sqlchemy.NotEquals(guestQ.Field("status"), api.VM_READY)).GroupBy(q.Field("numa_node")) - q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", q.Field("numa_node"))) + q = q.GroupBy(isq.Field("numa_node")) + q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", isq.Field("numa_node"))) subQ := q.SubQuery() numaQ := subQ.Query(subQ.Field("numa_node"), subQ.Field("numa_node_dev_count")) @@ -1803,11 +1829,11 @@ func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(d return stats, nil } -func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool, error) { +func (host *SHost) VirtualDeviceNumaBalance(devModel string, numaNode int8) (bool, error) { if numaNode < 0 { return true, nil } - stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devType, host.Id) + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devModel, host.Id) if err != nil { return true, err } @@ -1826,3 +1852,588 @@ func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool } return true, nil } + +func (manager *SIsolatedDeviceManager) GetAvailableIsolatedDeviceQuery(isq *sqlchemy.SQuery) *sqlchemy.SQuery { + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() + guestIsQ := guestIdevs.Query( + guestIdevs.Field("isolated_device_id"), + sqlchemy.SUM("memory_allocated", guestIdevs.Field("device_memory_size")), + sqlchemy.COUNT("guest_count", guestIdevs.Field("guest_id")), + ).GroupBy("isolated_device_id").SubQuery() + + if isq == nil { + isq = manager.Query() + } + + isq = isq.LeftJoin(guestIsQ, sqlchemy.Equals(isq.Field("id"), guestIsQ.Field("isolated_device_id"))) + cond1 := sqlchemy.AND( + sqlchemy.Equals(isq.Field("sharing_mode"), api.DEVICE_SHARING_MODE_HAMI), + sqlchemy.OR( + sqlchemy.IsNull(guestIsQ.Field("memory_allocated")), + sqlchemy.GT(isq.Field("memory_size"), guestIsQ.Field("memory_allocated")), + ), + ) + cond2 := sqlchemy.AND( + sqlchemy.NotEquals(isq.Field("sharing_mode"), api.DEVICE_SHARING_MODE_HAMI), + sqlchemy.OR( + sqlchemy.IsNull(guestIsQ.Field("guest_count")), + sqlchemy.GT(isq.Field("virtual_num"), guestIsQ.Field("guest_count")), + ), + ) + + isq = isq.Filter(sqlchemy.OR(cond1, cond2)) + + return isq +} + +type IsolatedDeviceAllocateStat struct { + SIsolatedDevice + + GuestCount int + MemoryAllocated int +} + +func (manager *SIsolatedDeviceManager) GetHostsIsolatedDeviceStats(hostIds []string) []IsolatedDeviceAllocateStat { + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() + guestIsQ := guestIdevs.Query( + guestIdevs.Field("isolated_device_id"), + sqlchemy.SUM("memory_allocated", guestIdevs.Field("device_memory_size")), + sqlchemy.COUNT("guest_count", guestIdevs.Field("guest_id")), + ).GroupBy("isolated_device_id").SubQuery() + + isq := manager.Query().In("host_id", hostIds) + isq = isq.LeftJoin(guestIsQ, sqlchemy.Equals(isq.Field("id"), guestIsQ.Field("isolated_device_id"))) + isq.AppendField(isq.QueryFields()...) + isq.AppendField(guestIsQ.Field("memory_allocated"), guestIsQ.Field("guest_count")) + stats := make([]IsolatedDeviceAllocateStat, 0) + err := isq.All(&stats) + if err != nil { + log.Errorf("GetHostsIsolatedDevicesDetails %s", err) + } + + return stats +} + +func (manager *SIsolatedDeviceManager) GetHostsGuestIsolatedDevices(hostIds []string) map[string][]string { + gidq := GuestIsolatedDeviceManager.Query().SubQuery() + isq := manager.Query().SubQuery() + + q := gidq.Query() + q = q.Join(isq, sqlchemy.Equals(isq.Field("id"), gidq.Field("isolated_device_id"))) + q = q.Filter(sqlchemy.In(isq.Field("host_id"), hostIds)) + + result := []struct { + IsolatedDeviceId string + GuestId string + }{} + err := q.All(&result) + if err != nil { + log.Errorf("GetHostsGuestIsolatedDevices query %s", err) + return nil + } + ret := map[string][]string{} + for i := range result { + if guests, ok := ret[result[i].IsolatedDeviceId]; ok { + ret[result[i].IsolatedDeviceId] = append(guests, result[i].GuestId) + } else { + ret[result[i].IsolatedDeviceId] = []string{result[i].GuestId} + } + } + return ret +} + +func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool { + if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI { + cnt, err := dev.getAllocatedCount() + if err != nil { + log.Errorf("failed getAllocatedCount %s", err) + return false + } + return dev.VirtualNum > cnt + } else { + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return false + } + return (dev.MemorySize - allocated) >= memoryRequest + } +} + +func (dev *SIsolatedDevice) IsFull() bool { + if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI { + cnt, err := dev.getAllocatedCount() + if err != nil { + log.Errorf("failed getAllocatedCount %s", err) + return true + } + return dev.VirtualNum <= cnt + } else { + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return true + } + return dev.MemorySize <= allocated + } +} + +func (manager *SIsolatedDeviceManager) InitializeData() error { + ctx := context.Background() + inited, err := manager.isInitializeDataDone() + if err != nil { + return errors.Wrap(err, "isInitializeDataDone") + } + if inited { + return nil + } + + if err := manager.migrateGuestIsolatedDevices(); err != nil { + return errors.Wrap(err, "migrateGuestIsolatedDevices") + } + if err := manager.mergeVirtualIsolatedDevices(); err != nil { + return errors.Wrap(err, "mergeVirtualIsolatedDevices") + } + if err := manager.initVirtualNum(); err != nil { + return errors.Wrap(err, "initVirtualNum") + } + if err := manager.migrateDevType(); err != nil { + return errors.Wrap(err, "migrateDevType") + } + if err := manager.markInitializeDataDone(ctx); err != nil { + return errors.Wrap(err, "markInitializeDataDone") + } + return nil +} + +func (manager *SIsolatedDeviceManager) getInitializeDataMetadataId() string { + return fmt.Sprintf("%s%s%s", isolatedDeviceInitializeDataObjType, db.OBJECT_TYPE_ID_SEP, isolatedDeviceInitializeDataObjId) +} + +func (manager *SIsolatedDeviceManager) isInitializeDataDone() (bool, error) { + md := db.SMetadata{} + err := db.Metadata.RawQuery("value", "deleted"). + Equals("id", manager.getInitializeDataMetadataId()). + Equals("key", isolatedDeviceInitializeDataKey). + First(&md) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return false, nil + } + return false, err + } + return !md.Deleted && md.Value == "true", nil +} + +func (manager *SIsolatedDeviceManager) markInitializeDataDone(ctx context.Context) error { + md := db.SMetadata{} + md.SetModelManager(db.Metadata, &md) + err := db.Metadata.RawQuery(). + Equals("id", manager.getInitializeDataMetadataId()). + Equals("key", isolatedDeviceInitializeDataKey). + First(&md) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return err + } + md.ObjType = isolatedDeviceInitializeDataObjType + md.ObjId = isolatedDeviceInitializeDataObjId + md.Id = manager.getInitializeDataMetadataId() + md.Key = isolatedDeviceInitializeDataKey + md.Value = "true" + return db.Metadata.TableSpec().Insert(ctx, &md) + } + _, err = db.Update(&md, func() error { + md.ObjType = isolatedDeviceInitializeDataObjType + md.ObjId = isolatedDeviceInitializeDataObjId + md.Value = "true" + md.Deleted = false + return nil + }) + return err +} + +func getIsolatedDeviceBaseAddr(addr string) string { + return strings.Split(addr, "-")[0] +} + +func isMergeableVirtualDevType(devType string) bool { + return utils.IsInStringArray(devType, api.VITRUAL_DEVICE_TYPES) +} + +type isolatedDeviceMergeKey struct { + HostId string + Addr string + MdevId string + VendorDeviceId string + DevType string +} + +func (manager *SIsolatedDeviceManager) migrateGuestIsolatedDevices() error { + rows, err := sqlchemy.GetDB().Query(fmt.Sprintf("SELECT id, guest_id, dev_type, network_index, disk_index FROM %s WHERE deleted = 0 AND guest_id IS NOT NULL AND LENGTH(guest_id) > 0", manager.TableSpec().Name())) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return errors.Wrap(err, "migrateGuestIsolatedDevices QueryRows") + } + if err != nil && errors.Cause(err) == sql.ErrNoRows { + return nil + } + defer rows.Close() + + ctx := context.Background() + migrated := 0 + guestIndexMap := map[string]int8{} + for rows.Next() { + var devId, guestId, devType string + var networkIndex int + var diskIndex int8 + if err = rows.Scan(&devId, &guestId, &devType, &networkIndex, &diskIndex); err != nil { + return errors.Wrap(err, "migrateGuestIsolatedDevices Scan") + } + cnt, err := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", guestId). + Equals("isolated_device_id", devId). + CountWithError() + if err != nil { + return errors.Wrapf(err, "count guest isolated device for device %s", devId) + } + if cnt == 0 { + idx, ok := guestIndexMap[guestId] + if !ok { + maxIdx, err := manager.getGuestIsolatedDeviceMaxIndex(guestId) + if err != nil { + return errors.Wrapf(err, "getGuestIsolatedDeviceMaxIndex guest %s", guestId) + } + idx = maxIdx + 1 + } + guestIndexMap[guestId] = idx + 1 + + guestIsolatedDevice := SGuestIsolatedDevice{} + guestIsolatedDevice.SetModelManager(GuestIsolatedDeviceManager, &guestIsolatedDevice) + guestIsolatedDevice.GuestId = guestId + guestIsolatedDevice.IsolatedDeviceId = devId + guestIsolatedDevice.Index = idx + if devType == api.GPU_VGA_TYPE { + guestIsolatedDevice.GpuType = api.GPU_VGA + } else if utils.IsInStringArray(devType, api.GPU_TYPES) { + guestIsolatedDevice.GpuType = api.GPU_HPC + } + if networkIndex >= 0 { + guestIsolatedDevice.NetworkIndex = networkIndex + } + if diskIndex >= 0 { + guestIsolatedDevice.DiskIndex = diskIndex + } + if err := GuestIsolatedDeviceManager.TableSpec().Insert(ctx, &guestIsolatedDevice); err != nil { + return errors.Wrapf(err, "insert guest isolated device for device %s guest %s", devId, guestId) + } + migrated++ + } + } + + log.Infof("migrated %d legacy isolated device guest assign to guest_isolated_devices_tbl", migrated) + return nil +} + +func (manager *SIsolatedDeviceManager) getGuestIsolatedDeviceMaxIndex(guestId string) (int8, error) { + type maxIdxResult struct { + MaxIndex int8 + } + sq := GuestIsolatedDeviceManager.Query().Equals("guest_id", guestId).SubQuery() + q := sq.Query(sqlchemy.MAX("max_index", sq.Field("index"))) + ret := maxIdxResult{MaxIndex: -1} + err := q.First(&ret) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return -1, err + } + return ret.MaxIndex, nil +} + +func (manager *SIsolatedDeviceManager) mergeVirtualIsolatedDevices() error { + devs := make([]SIsolatedDevice, 0) + q := manager.Query().In("dev_type", api.VITRUAL_DEVICE_TYPES).NotEquals("dev_type", api.CONTAINER_DEV_NVIDIA_HAMI) + err := db.FetchModelObjects(manager, q, &devs) + if err != nil { + return errors.Wrap(err, "FetchModelObjects") + } + if len(devs) == 0 { + return nil + } + + grouped := map[isolatedDeviceMergeKey][]*SIsolatedDevice{} + for i := range devs { + dev := &devs[i] + key := isolatedDeviceMergeKey{ + HostId: dev.HostId, + Addr: getIsolatedDeviceBaseAddr(dev.Addr), + MdevId: dev.MdevId, + VendorDeviceId: dev.VendorDeviceId, + DevType: dev.DevType, + } + grouped[key] = append(grouped[key], dev) + } + + mergedGroups := 0 + deletedDevs := 0 + for _, group := range grouped { + if len(group) <= 1 { + continue + } + sort.Slice(group, func(i, j int) bool { + return group[i].CreatedAt.Before(group[j].CreatedAt) + }) + keeper := group[0] + ids := make([]string, len(group)-1) + for i := 1; i < len(group); i++ { + ids[i-1] = group[i].Id + } + err := manager.doMergeGuestIsolatedDevices(keeper.Id, getIsolatedDeviceBaseAddr(keeper.Addr), ids) + if err != nil { + return err + } + host := HostManager.FetchHostById(group[0].HostId) + if host != nil && host.HostType == api.HOST_TYPE_CONTAINER { + err = manager.doReplaceContainerIsolatedDeviceId(keeper.Id, ids) + if err != nil { + return err + } + } + mergedGroups++ + } + log.Infof("merged %d duplicate virtual isolated device groups, deleted %d duplicate devices", mergedGroups, deletedDevs) + return nil +} + +func (manager *SIsolatedDeviceManager) doReplaceContainerIsolatedDeviceId(keeperId string, originIds []string) error { + log.Infof("start replace contaienr isolated device id, originIds %v, keeper %s", originIds, keeperId) + ids := []string{keeperId} + ids = append(ids, originIds...) + gdevs := make([]SGuestIsolatedDevice, 0) + q := GuestIsolatedDeviceManager.Query().In("isolated_device_id", ids) + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return errors.Wrap(err, "GuestIsolatedDeviceManager.FetchModelObjects") + } + for i := range gdevs { + ctrs, err := GetContainerManager().GetContainersByPod(gdevs[i].GuestId) + if err != nil { + return errors.Wrapf(err, "GetContainerManager().GetContainersByPod") + } + for j := range ctrs { + ctrPtr := &ctrs[j] + + spec := new(api.ContainerSpec) + if err := jsonutils.Marshal(ctrPtr.Spec).Unmarshal(spec); err != nil { + return errors.Wrap(err, "deep copy spec") + } + + updated := false + for k := range spec.Devices { + if spec.Devices[k].IsolatedDevice == nil { + continue + } + if !utils.IsInStringArray(spec.Devices[k].IsolatedDevice.Id, ids) { + continue + } + spec.Devices[k].IsolatedDevice.Id = keeperId + spec.Devices[k].IsolatedDevice.GuestIsolatedDeviceIndex = int(gdevs[i].Index) + updated = true + } + if !updated { + continue + } + _, err = db.Update(ctrPtr, func() error { + ctrPtr.Spec = spec + return nil + }) + if err != nil { + return errors.Wrap(err, "update ctr isolated device id") + } + log.Infof("replace container isolated device id %s to %s", ctrPtr.Id, keeperId) + } + } + return nil +} + +func (manager *SIsolatedDeviceManager) doMergeGuestIsolatedDevices(keeperId, keeperNewAddr string, originIds []string) error { + tx, err := sqlchemy.GetDB().Begin() + if err != nil { + return errors.Wrap(err, "failed begin TRANSACTION") + } + defer func() { + if err != nil { + tx.Rollback() + } + }() + buildInPlaceholders := func(n int) string { + parts := make([]string, n) + for i := range parts { + parts[i] = "?" + } + return strings.Join(parts, ",") + } + var res sql.Result + + if len(originIds) > 0 { + sql := fmt.Sprintf( + "update %s set isolated_device_id = ? where isolated_device_id in (%s)", + GuestIsolatedDeviceManager.TableSpec().Name(), buildInPlaceholders(len(originIds)), + ) + args := make([]interface{}, 1, 1+len(originIds)) + args[0] = keeperId + for i := range originIds { + args = append(args, originIds[i]) + } + res, err = tx.Exec(sql, args...) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ := res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + + sql = fmt.Sprintf( + "update %s set deleted = 1 where id in (%s)", + IsolatedDeviceManager.TableSpec().Name(), buildInPlaceholders(len(originIds)), + ) + args = make([]interface{}, len(originIds)) + for i := range originIds { + args[i] = originIds[i] + } + res, err = tx.Exec(sql, args...) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ = res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + if affected != int64(len(originIds)) { + return errors.Errorf("TRANSACTION: %s affected rows %d not equal to originIds length", sql, affected) + } + } + + virtualNum := 1 + len(originIds) + sql := fmt.Sprintf( + "update %s set virtual_num = ?, addr = ? where id = ?", + IsolatedDeviceManager.TableSpec().Name(), + ) + res, err = tx.Exec(sql, virtualNum, keeperNewAddr, keeperId) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ := res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + + if err = tx.Commit(); err != nil { + return errors.Wrap(err, "failed commit TRANSACTION") + } + return nil +} + +func (manager *SIsolatedDeviceManager) initVirtualNum() error { + devs := make([]SIsolatedDevice, 0) + q := manager.Query().NotEquals("dev_type", api.CONTAINER_DEV_NVIDIA_HAMI) + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("virtual_num")), + sqlchemy.LE(q.Field("virtual_num"), 0), + )) + err := db.FetchModelObjects(manager, q, &devs) + if err != nil { + return errors.Wrap(err, "FetchModelObjects") + } + updated := 0 + for i := range devs { + dev := &devs[i] + virtualNum := 1 + if isMergeableVirtualDevType(dev.DevType) { + cnt, err := dev.getAllocatedCount() + if err != nil { + return errors.Wrapf(err, "getAllocatedCount device %s", dev.Id) + } + if cnt > virtualNum { + virtualNum = cnt + } + } + if _, err := db.Update(dev, func() error { + dev.VirtualNum = virtualNum + return nil + }); err != nil { + return errors.Wrapf(err, "set virtual_num on device %s", dev.Id) + } + updated++ + } + log.Infof("initialized virtual_num for %d isolated devices", updated) + return nil +} + +func (manager *SIsolatedDeviceManager) migrateDevType() error { + hotPluggableDevTypes := []string{ + api.DIRECT_PCI_TYPE, api.USB_TYPE, api.GPU_VGA_TYPE, api.GPU_HPC_TYPE, + api.SRIOV_VGPU_TYPE, api.LEGACY_VGPU_TYPE, + } + + for _, devType := range api.VALID_PASSTHROUGH_TYPES { + var sharingMode string + switch devType { + case api.DIRECT_PCI_TYPE, api.USB_TYPE, api.GPU_VGA_TYPE, api.GPU_HPC_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_EXCLUSIVE + case api.SRIOV_VGPU_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_SRIOV + case api.CONTAINER_DEV_NVIDIA_MPS: + sharingMode = api.DEVICE_SHARING_MODE_MPS + case api.CONTAINER_DEV_NVIDIA_HAMI: + sharingMode = api.DEVICE_SHARING_MODE_HAMI + case api.LEGACY_VGPU_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_MDEV + default: + sharingMode = api.DEVICE_SHARING_MODE_UNLIMITED + } + var hotPluggable = 0 + if utils.IsInStringArray(devType, hotPluggableDevTypes) { + hotPluggable = 1 + } + var targetDevType string + switch { + case utils.IsInStringArray(devType, api.GPU_TYPES): + targetDevType = api.GPU_TYPE + case utils.IsInStringArray(devType, api.NETINT_TYPES): + targetDevType = api.NETINT_TYPE + case devType == api.CONTAINER_DEV_CPH_AOSP_BINDER: + targetDevType = api.BINDER_TYPE + case devType == api.CONTAINER_DEV_ASCEND_NPU: + targetDevType = api.NPU_TYPE + default: + targetDevType = devType + } + + sql := fmt.Sprintf( + "update %s set dev_type = ?, sharing_mode = ?, hot_pluggable = ? where dev_type = ? and deleted = 0", + manager.TableSpec().Name(), + ) + res, err := sqlchemy.GetDB().Exec(sql, targetDevType, sharingMode, hotPluggable, devType) + if err != nil { + return errors.Wrapf(err, "update dev_type from %v to %s", devType, targetDevType) + } + effects, _ := res.RowsAffected() + log.Infof("updated dev_type from %v to %s, effects: %d", devType, targetDevType, effects) + } + return nil +} + +func (dev *SIsolatedDevice) IsValidAttachDev() bool { + if dev.DevType == api.NIC_TYPE || dev.DevType == api.NVME_PT_TYPE { + return false + } + return true +} + +func (dev *SIsolatedDevice) IsKvmExclusiveGPU() bool { + if dev.DevType != api.GPU_TYPE { + return false + } + if dev.SharingMode != api.DEVICE_SHARING_MODE_EXCLUSIVE { + return false + } + host := dev.GetHost() + if host.HostType != api.HOST_TYPE_KVM { + return false + } + return true +} diff --git a/pkg/compute/models/isolateddeviceresouce.go b/pkg/compute/models/isolateddeviceresouce.go new file mode 100644 index 0000000000..51d4572aae --- /dev/null +++ b/pkg/compute/models/isolateddeviceresouce.go @@ -0,0 +1,198 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +type SIsolatedDeviceResourceBase struct { + IsolatedDeviceId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +type SIsolatedDeviceResourceBaseManager struct { +} + +func (manager *SIsolatedDeviceResourceBaseManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.IsolateDeviceDetails { + rows := make([]api.IsolateDeviceDetails, len(objs)) + devIds := make([]string, len(objs)) + for i := range objs { + var base *SIsolatedDeviceResourceBase + err := reflectutils.FindAnonymouStructPointer(objs[i], &base) + if err != nil { + log.Errorf("Cannot find SIsolatedDeviceResourceBase in object %s", objs[i]) + continue + } + devIds[i] = base.IsolatedDeviceId + } + + devMap := make(map[string]SIsolatedDevice) + err := db.FetchStandaloneObjectsByIds(IsolatedDeviceManager, devIds, devMap) + if err != nil { + log.Errorf("FetchStandaloneObjectsByIds fail %s", err) + return rows + } + + devObjs := make([]interface{}, len(objs)) + devs := make([]SIsolatedDevice, len(objs)) + for i := range rows { + if dev, ok := devMap[devIds[i]]; ok { + devs[i] = dev + if err := jsonutils.Marshal(dev).Unmarshal(&rows[i].SIsolatedDevice); err != nil { + log.Errorf("unmarshal isolated device %s details failed %s", dev.Id, err) + } + } + devObjs[i] = &devs[i] + } + + devRows := IsolatedDeviceManager.FetchCustomizeColumns(ctx, userCred, query, devObjs, fields, isList) + for i := range rows { + rows[i].StandaloneResourceDetails = devRows[i].StandaloneResourceDetails + rows[i].SharableResourceBaseInfo = devRows[i].SharableResourceBaseInfo + rows[i].HostResourceInfo = devRows[i].HostResourceInfo + rows[i].Guest = devRows[i].Guest + rows[i].GuestStatus = devRows[i].GuestStatus + } + return rows +} + +func (manager *SIsolatedDeviceResourceBaseManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + devQ := IsolatedDeviceManager.Query("id") + devQ, err := IsolatedDeviceManager.ListItemFilter(ctx, devQ, userCred, query.IsolatedDeviceListInput) + if err != nil { + return nil, errors.Wrap(err, "IsolatedDeviceManager.ListItemFilter") + } + q = q.Filter(sqlchemy.In(q.Field("isolated_device_id"), devQ.SubQuery())) + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + switch field { + case "isolated_device": + devQ := IsolatedDeviceManager.Query("name", "id").SubQuery() + q = q.AppendField(devQ.Field("name", field)).Distinct() + q = q.Join(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + return q, nil + case "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node": + devQ := IsolatedDeviceManager.Query(field, "id").SubQuery() + q = q.AppendField(devQ.Field(field)).Distinct() + q = q.Join(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + return q, nil + default: + devQ := IsolatedDeviceManager.Query("id", "host_id").SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + q, err := IsolatedDeviceManager.SHostResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + } + return q, httperrors.ErrNotFound +} + +func (manager *SIsolatedDeviceResourceBaseManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, nil + } + orderQ := IsolatedDeviceManager.Query("id") + orderSubQ := orderQ.SubQuery() + orderQ, orders, fields := manager.GetOrderBySubQuery(orderQ, orderSubQ, orderQ.Field("id"), userCred, query, nil, nil) + q = q.LeftJoin(orderSubQ, sqlchemy.Equals(q.Field("isolated_device_id"), orderSubQ.Field("id"))) + q = db.OrderByFields(q, orders, fields) + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetOrderBySubQuery( + q *sqlchemy.SQuery, + subq *sqlchemy.SSubQuery, + joinField sqlchemy.IQueryField, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, + orders []string, + fields []sqlchemy.IQueryField, +) (*sqlchemy.SQuery, []string, []sqlchemy.IQueryField) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, orders, fields + } + devQ := IsolatedDeviceManager.Query().SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(joinField, devQ.Field("id"))) + q, orders, fields = IsolatedDeviceManager.SHostResourceBaseManager.GetOrderBySubQuery(q, subq, devQ.Field("host_id"), userCred, query.HostFilterListInput, orders, fields) + return q, orders, fields +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetOrderByFields(query api.GuestIsolatedDeviceListInput) []string { + return IsolatedDeviceManager.SHostResourceBaseManager.GetOrderByFields(query.HostFilterListInput) +} + +func (manager *SIsolatedDeviceResourceBaseManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + if keys.ContainsAny(manager.GetExportKeys()...) { + devQ := IsolatedDeviceManager.Query("id", "name", "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node", "host_id").SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + if keys.Contains("isolated_device") { + q = q.AppendField(devQ.Field("name", "isolated_device")) + } + for _, key := range []string{"dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node"} { + if keys.Contains(key) { + q = q.AppendField(devQ.Field(key)) + } + } + if keys.ContainsAny(IsolatedDeviceManager.SHostResourceBaseManager.GetExportKeys()...) { + var err error + q, err = IsolatedDeviceManager.SHostResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SHostResourceBaseManager.ListItemExportKeys") + } + } + } + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetExportKeys() []string { + keys := []string{"isolated_device", "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node"} + keys = append(keys, IsolatedDeviceManager.SHostResourceBaseManager.GetExportKeys()...) + return keys +} diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 47705e506e..f279a5a645 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -288,6 +288,7 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { models.GuestsecgroupManager, models.LoadbalancernetworkManager, models.GuestdiskManager, + models.GuestIsolatedDeviceManager, models.GroupnetworkManager, models.GroupguestManager, models.StoragecachedimageManager, diff --git a/pkg/compute/specs/handler.go b/pkg/compute/specs/handler.go index 55f99a9ca6..ac86ceb4eb 100644 --- a/pkg/compute/specs/handler.go +++ b/pkg/compute/specs/handler.go @@ -140,7 +140,7 @@ func queryHosts( } if len(gpuModels) != 0 { - devs, err := models.IsolatedDeviceManager.FindUnusedByModels(gpuModels) + devs, err := models.IsolatedDeviceManager.FindAvailableByModels(gpuModels) if err != nil { return nil, err } @@ -154,7 +154,7 @@ func queryHosts( if !sets.NewString(gpuHostIds...).Has(obj.GetId()) { return false } - gpus, _ := models.IsolatedDeviceManager.FindUnusedGpusOnHost(obj.GetId()) + gpus, _ := models.IsolatedDeviceManager.FindAvailableGpusOnHost(obj.GetId()) if len(gpus) == 0 { return false } diff --git a/pkg/compute/tasks/guest/guest_live_migrate_task.go b/pkg/compute/tasks/guest/guest_live_migrate_task.go index 5a951d0e33..3412a1a235 100644 --- a/pkg/compute/tasks/guest/guest_live_migrate_task.go +++ b/pkg/compute/tasks/guest/guest_live_migrate_task.go @@ -91,11 +91,12 @@ func (task *GuestMigrateTask) GetSchedParams() (*schedapi.ScheduleInput, error) } res := guest.GetSchedMigrateParams(task.GetUserCred(), input) - if devs, _ := guest.GetIsolatedDevices(); len(devs) > 0 { + if devs, _ := guest.GetGuestIsolatedDevices(); len(devs) > 0 { preferNumaNodesSet := cpuset.NewBuilder() for i := range devs { - if devs[i].NumaNode >= 0 { - preferNumaNodesSet.Add(int(devs[i].NumaNode)) + dev := devs[i].GetIsolatedDevice() + if dev.NumaNode >= 0 { + preferNumaNodesSet.Add(int(dev.NumaNode)) } } res.PreferNumaNodes = preferNumaNodesSet.Result().ToSlice() diff --git a/pkg/compute/tasks/guest/guest_start_task.go b/pkg/compute/tasks/guest/guest_start_task.go index 4aae5bf454..6312fc8fe7 100644 --- a/pkg/compute/tasks/guest/guest_start_task.go +++ b/pkg/compute/tasks/guest/guest_start_task.go @@ -53,18 +53,32 @@ func (self *GuestStartTask) attachReleasedDevices(ctx context.Context, guest *mo if len(devs) == 0 { return self.ScheduleRun(nil) } - attachReq := make(map[string]int) + + type DevRequest struct { + Model string + GpuType string + SharingMode string + MemoryRequest int + } + + devGroup := map[DevRequest]int{} for _, dev := range devs { - count, ok := attachReq[dev.Model] + devReq := DevRequest{dev.Model, dev.GpuType, dev.SharingMode, dev.MemoryRequest} + + cnt, ok := devGroup[devReq] if !ok { - attachReq[dev.Model] = 1 + devGroup[devReq] = 1 } else { - attachReq[dev.Model] = count + 1 + devGroup[devReq] = cnt + 1 } } - if err := guest.AttachIsolatedDevices(ctx, self.GetUserCred(), attachReq); err != nil { - return errors.Wrap(err, "attach isolated devices") + for devReq, cnt := range devGroup { + attachReq := map[string]int{devReq.Model: cnt} + if err := guest.AttachIsolatedDevices(ctx, self.GetUserCred(), attachReq, devReq.GpuType, devReq.SharingMode, &devReq.MemoryRequest); err != nil { + return errors.Wrap(err, "attach isolated devices") + } } + return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId()) } diff --git a/pkg/compute/tasks/guest/guest_stop_task.go b/pkg/compute/tasks/guest/guest_stop_task.go index 1d4226d103..7543606e06 100644 --- a/pkg/compute/tasks/guest/guest_stop_task.go +++ b/pkg/compute/tasks/guest/guest_stop_task.go @@ -93,25 +93,18 @@ func (self *GuestStopTask) releaseDevices(ctx context.Context, guest *models.SGu if guest.ShutdownBehavior != api.SHUTDOWN_STOP_RELEASE_GPU { return self.ScheduleRun(nil) } - devs, err := guest.GetIsolatedDevices() + devs, err := guest.GetGuestGpuIsolatedDevices() if err != nil { return errors.Wrapf(err, "GetIsolatedDevices of guest %s", guest.GetId()) } - gpus := make([]models.SIsolatedDevice, 0) - for _, dev := range devs { - if dev.IsGPU() { - tmpDev := dev - gpus = append(gpus, tmpDev) - } - } - if len(gpus) == 0 { + if len(devs) == 0 { return self.ScheduleRun(nil) } - if err := guest.SetReleasedIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil { + if err := guest.SetReleasedIsolatedDevices(ctx, self.GetUserCred(), devs); err != nil { return errors.Wrapf(err, "SetReleasedIsolatedDevices of guest %s", guest.GetId()) } - if err := guest.DetachIsolatedDevices(ctx, self.GetUserCred(), gpus); err != nil { + if err := guest.DetachIsolatedDevices(ctx, self.GetUserCred(), devs); err != nil { return errors.Wrapf(err, "DetachIsolatedDevices of guest %s", guest.GetId()) } return guest.StartIsolatedDevicesSyncTask(ctx, self.GetUserCred(), false, self.GetTaskId()) diff --git a/pkg/hostman/container/device/isolated_device.go b/pkg/hostman/container/device/isolated_device.go index 85f3841418..a563855846 100644 --- a/pkg/hostman/container/device/isolated_device.go +++ b/pkg/hostman/container/device/isolated_device.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/isolated_device" ) @@ -42,28 +43,29 @@ func (i isolatedDevice) GetRuntimeDevices(input *hostapi.ContainerCreateInput, d if len(devs) == 0 { return nil, nil } - devsMap := map[string][]*hostapi.ContainerDevice{} + devsMap := map[isolated_device.IContainerDeviceManager][]*hostapi.ContainerDevice{} for _, dev := range devs { - if mapDevs, ok := devsMap[dev.IsolatedDevice.DeviceType]; ok { - devsMap[dev.IsolatedDevice.DeviceType] = append(mapDevs, dev) + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return nil, errors.Wrapf(errors.ErrNotFound, "device %s not exist", dev.IsolatedDevice.Id) + } + devMan := iDev.GetContainerDeviceManager() + + if mapDevs, ok := devsMap[devMan]; ok { + devsMap[devMan] = append(mapDevs, dev) } else { - devsMap[dev.IsolatedDevice.DeviceType] = []*hostapi.ContainerDevice{dev} + devsMap[devMan] = []*hostapi.ContainerDevice{dev} } } ret := make([]*runtimeapi.Device, 0) - for devType, mappedDevs := range devsMap { - man, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceType(devType)) - if err != nil { - return nil, errors.Wrapf(err, "GetContainerDeviceManager by type %q", devType) - } - + for devMan, mappedDevs := range devsMap { for idx := range mappedDevs { mDev := mappedDevs[idx] if mDev.IsolatedDevice != nil && mDev.IsolatedDevice.OnlyEnv != nil { continue } - ctrDevs, commonDevs, err := man.NewContainerDevices(input, mappedDevs[idx]) + ctrDevs, commonDevs, err := devMan.NewContainerDevices(input, mappedDevs[idx]) if err != nil { return nil, errors.Wrapf(err, "NewContainerDevices with %#v", devs) } diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 108f413713..dc0fdf0728 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -1043,7 +1043,7 @@ func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev *desc.SGuestIsolatedDevice) id := devObj.GetQemuId() dev.VfioDevs = make([]*desc.VFIODevice, 0) vfioDev := desc.NewVfioDevice( - *cType, "vfio-pci", id, devObj.GetAddr(), dev.DevType == api.GPU_VGA_TYPE, + *cType, "vfio-pci", id, devObj.GetAddr(), dev.GpuType == api.GPU_VGA, ) dev.VfioDevs = append(dev.VfioDevs, vfioDev) diff --git a/pkg/hostman/guestman/pci.go b/pkg/hostman/guestman/pci.go index 111ee905f1..a7a4feadf4 100644 --- a/pkg/hostman/guestman/pci.go +++ b/pkg/hostman/guestman/pci.go @@ -22,11 +22,11 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/guestman/qemu" "yunion.io/x/onecloud/pkg/hostman/monitor" "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/util/fileutils2" ) @@ -345,14 +345,14 @@ func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCICont manager := s.manager.GetHost().GetIsolatedDeviceManager() for i := 0; i < len(s.Desc.IsolatedDevices); i++ { dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr) - if s.Desc.IsolatedDevices[i].DevType == api.USB_TYPE { + if s.Desc.IsolatedDevices[i].DevType == compute.USB_TYPE { s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", dev.GetQemuId()) s.Desc.IsolatedDevices[i].Usb.Options = dev.GetPassthroughOptions() } else { id := dev.GetQemuId() s.Desc.IsolatedDevices[i].VfioDevs = make([]*desc.VFIODevice, 0) vfioDev := desc.NewVfioDevice( - *cType, "vfio-pci", id, dev.GetAddr(), s.Desc.IsolatedDevices[i].DevType == api.GPU_VGA_TYPE, + *cType, "vfio-pci", id, dev.GetAddr(), s.Desc.IsolatedDevices[i].GpuType == compute.GPU_VGA, ) s.Desc.IsolatedDevices[i].VfioDevs = append(s.Desc.IsolatedDevices[i].VfioDevs, vfioDev) diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index d76c4e4830..8a6f39d705 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -37,7 +37,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/sets" - "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" @@ -838,7 +837,8 @@ func (s *sPodGuestInstance) GetPodContainerCriIds() []string { func (s *sPodGuestInstance) HasContainerNvidiaGpu() bool { for i := range s.Desc.IsolatedDevices { - if utils.IsInStringArray(s.Desc.IsolatedDevices[i].DevType, computeapi.NVIDIA_GPU_TYPES) { + vendor := strings.Split(s.Desc.IsolatedDevices[i].VendorDeviceId, ":")[0] + if vendor == computeapi.NVIDIA_VENDOR_ID { return true } } @@ -2340,39 +2340,6 @@ func (s *sPodGuestInstance) getEtcFilesMount(ctrId string) ([]*runtimeapi.Mount, return []*runtimeapi.Mount{etcHostsMount, etcHostnameMount, etcResolvConfMount}, nil } -type FilteredContainerDevices struct { - EnvDevs []*hostapi.ContainerDevice - CDIDevs []*hostapi.ContainerDevice - RestDevs []*hostapi.ContainerDevice -} - -func filterContainerIsolatedDevices(devs []*hostapi.ContainerDevice, devTypes sets.String) FilteredContainerDevices { - envDevs := []*hostapi.ContainerDevice{} - restDevs := []*hostapi.ContainerDevice{} - cdiDevs := []*hostapi.ContainerDevice{} - for i := range devs { - dev := devs[i] - if dev.IsolatedDevice != nil { - devType := dev.IsolatedDevice.DeviceType - if !devTypes.Has(devType) { - continue - } - if dev.IsolatedDevice.IsCDIUsed() { - cdiDevs = append(cdiDevs, dev) - } else if len(dev.IsolatedDevice.OnlyEnv) > 0 { - envDevs = append(envDevs, dev) - } else { - restDevs = append(restDevs, dev) - } - } - } - return FilteredContainerDevices{ - EnvDevs: envDevs, - CDIDevs: cdiDevs, - RestDevs: restDevs, - } -} - func getEnvsFromDevices(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue { retEnvs := []*runtimeapi.KeyValue{} for _, dev := range devs { @@ -2411,32 +2378,48 @@ func getEnvsFromDevices(devs []*hostapi.ContainerDevice) []*runtimeapi.KeyValue } func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.ContainerSpec, ctrCfg *runtimeapi.ContainerConfig) error { - devTypes := sets.NewString( - string(isolated_device.ContainerDeviceTypeNvidiaGpu), - string(isolated_device.ContainerDeviceTypeNvidiaMps), - string(isolated_device.ContainerDeviceTypeNvidiaGpuShare), - string(isolated_device.ContainerDeviceTypeAscendNpu), - ) - fDevs := filterContainerIsolatedDevices(spec.Devices, devTypes) - if len(fDevs.EnvDevs) != 0 { - ctrCfg.Envs = append(ctrCfg.Envs, getEnvsFromDevices(fDevs.EnvDevs)...) - } - restDevsByType := map[string][]*hostapi.ContainerDevice{} - restDevTypeOrder := []string{} - for i := range fDevs.RestDevs { - dev := fDevs.RestDevs[i] + envDevs := []*hostapi.ContainerDevice{} + restDevs := []*hostapi.ContainerDevice{} + cdiDevs := []*hostapi.ContainerDevice{} + for i := range spec.Devices { + dev := spec.Devices[i] + if dev.IsolatedDevice == nil { + continue + } devType := dev.IsolatedDevice.DeviceType - if _, ok := restDevsByType[devType]; !ok { - restDevTypeOrder = append(restDevTypeOrder, devType) + if devType != computeapi.GPU_TYPE && devType != computeapi.NPU_TYPE { + continue + } + if dev.IsolatedDevice.IsCDIUsed() { + cdiDevs = append(cdiDevs, dev) + } else if len(dev.IsolatedDevice.OnlyEnv) > 0 { + envDevs = append(envDevs, dev) + } else { + restDevs = append(restDevs, dev) } - restDevsByType[devType] = append(restDevsByType[devType], dev) } - for _, devType := range restDevTypeOrder { - devs := restDevsByType[devType] - devMan, err := isolated_device.GetContainerDeviceManager(isolated_device.ContainerDeviceType(devType)) - if err != nil { - return errors.Wrapf(err, "GetContainerDeviceManager by type %q", devType) + + if len(envDevs) != 0 { + ctrCfg.Envs = append(ctrCfg.Envs, getEnvsFromDevices(envDevs)...) + } + restDevsByType := map[isolated_device.IContainerDeviceManager][]*hostapi.ContainerDevice{} + restDevTypeOrder := []isolated_device.IContainerDeviceManager{} + for i := range restDevs { + dev := restDevs[i] + + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + if iDev == nil { + return errors.Wrapf(errors.ErrNotFound, "device %s not exist", dev.IsolatedDevice.Id) } + devMan := iDev.GetContainerDeviceManager() + + if _, ok := restDevsByType[devMan]; !ok { + restDevTypeOrder = append(restDevTypeOrder, devMan) + } + restDevsByType[devMan] = append(restDevsByType[devMan], dev) + } + for _, devMan := range restDevTypeOrder { + devs := restDevsByType[devMan] envs, mounts := devMan.GetContainerExtraConfigures(devs) if len(envs) > 0 { ctrCfg.Envs = append(ctrCfg.Envs, envs...) @@ -2446,8 +2429,8 @@ func (s *sPodGuestInstance) getIsolatedDeviceExtraConfig(spec *hostapi.Container } } - if len(fDevs.CDIDevs) > 0 { - cdiDevs, err := isolated_device.GetContainerCDIDevices(fDevs.CDIDevs) + if len(cdiDevs) > 0 { + cdiDevs, err := isolated_device.GetContainerCDIDevices(cdiDevs) if err != nil { return errors.Wrap(err, "GetContainerCDIDevices") } diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 9705d05887..9bd6ed14e2 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -1159,10 +1159,8 @@ func (s *SKVMGuestInstance) startMemCleaner() error { } func (s *SKVMGuestInstance) gpusHasVga() bool { - manager := s.manager.GetHost().GetIsolatedDeviceManager() for i := 0; i < len(s.Desc.IsolatedDevices); i++ { - dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr) - if dev.GetDeviceType() == api.GPU_VGA_TYPE { + if s.Desc.IsolatedDevices[i].GpuType == api.GPU_VGA { return true } } @@ -1173,7 +1171,7 @@ func (s *SKVMGuestInstance) hasGPU() bool { manager := s.manager.GetHost().GetIsolatedDeviceManager() for i := 0; i < len(s.Desc.IsolatedDevices); i++ { dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr) - if dev.GetDeviceType() == api.GPU_VGA_TYPE || dev.GetDeviceType() == api.GPU_HPC_TYPE { + if dev.GetDeviceType() == api.GPU_TYPE { return true } } diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index 6e2a5fd259..385f90a587 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -38,6 +38,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostpinger" "yunion.io/x/onecloud/pkg/hostman/hostmetrics" "yunion.io/x/onecloud/pkg/hostman/hostutils" + _ "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/hostman/storageman/diskhandlers" diff --git a/pkg/hostman/hostinfo/container.go b/pkg/hostman/hostinfo/container.go index 3926426b24..c8bdafbf80 100644 --- a/pkg/hostman/hostinfo/container.go +++ b/pkg/hostman/hostinfo/container.go @@ -17,6 +17,7 @@ package hostinfo import ( "context" "path" + "strings" "time" "yunion.io/x/log" @@ -106,7 +107,8 @@ func (h *SHostInfo) HasContainerVastaitechGpu() bool { hasVastaitechGpus := false devs := h.IsolatedDeviceMan.GetDevices() for i := range devs { - if devs[i].GetDeviceType() == apis.CONTAINER_DEV_VASTAITECH_GPU { + vendorId := strings.Split(devs[i].GetVendorDeviceId(), ":")[0] + if utils.IsInStringArray(devs[i].GetSharingMode(), apis.VIRTUAL_SHARING_MODES) && vendorId == apis.VASTAITECH_VENDOR_ID { hasVastaitechGpus = true } } @@ -121,7 +123,8 @@ func (h *SHostInfo) HasContainerCphAmdGpu() bool { hasCphAmdGpus := false devs := h.IsolatedDeviceMan.GetDevices() for i := range devs { - if devs[i].GetDeviceType() == apis.CONTAINER_DEV_CPH_AMD_GPU { + vendorId := strings.Split(devs[i].GetVendorDeviceId(), ":")[0] + if utils.IsInStringArray(devs[i].GetSharingMode(), apis.VIRTUAL_SHARING_MODES) && vendorId == apis.AMD_VENDOR_ID { hasCphAmdGpus = true } } @@ -137,10 +140,12 @@ func (h *SHostInfo) HasContainerNvidiaGpu() bool { nvDevs := make([]isolated_device.IDevice, 0) devs := h.IsolatedDeviceMan.GetDevices() for i := range devs { - if utils.IsInStringArray(devs[i].GetDeviceType(), apis.NVIDIA_GPU_TYPES) { + vendorId := strings.Split(devs[i].GetVendorDeviceId(), ":")[0] + if utils.IsInStringArray(devs[i].GetSharingMode(), apis.VIRTUAL_SHARING_MODES) && vendorId == apis.NVIDIA_VENDOR_ID { hasNvidiaGpus = true nvDevs = append(nvDevs, devs[i]) } + } h.hasNvidiaGpus = &hasNvidiaGpus h.containerNvidiaGpus = nvDevs diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 463a6b4b7c..fbb4ec950b 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -59,7 +59,6 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostutils/hardware" "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/isolated_device" - _ "yunion.io/x/onecloud/pkg/hostman/isolated_device/container_device" "yunion.io/x/onecloud/pkg/hostman/monitor" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" @@ -2311,7 +2310,11 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { } } - enableDevWhitelist := options.HostOptions.EnableIsolatedDeviceWhitelist + _, err := modules.Hosts.GetSpecific(h.GetSession(), h.HostId, "guest-isolated-devices-initialized", nil) + if err != nil { + return nil, errors.Wrap(err, "check GuestIsolatedDevicesInitialized") + } + offloadNics, err := h.getNicsInterfaces(options.HostOptions.OvsOffloadNics) if err != nil { return nil, err @@ -2320,18 +2323,21 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { if err != nil { return nil, err } - h.IsolatedDeviceMan.ProbePCIDevices( - options.HostOptions.DisableGPU, - options.HostOptions.DisableUSB, - options.HostOptions.DisableCustomDevice, - sriovNics, offloadNics, - options.HostOptions.PTNVMEConfigs, - options.HostOptions.AMDVgpuPFs, - options.HostOptions.NVIDIAVgpuPFs, - options.HostOptions.EnableCudaMPS, - options.HostOptions.EnableContainerAscendNPU, - enableDevWhitelist, - ) + probeOpts := &isolated_device.SIsolatedDeviceProbeOptions{ + SkipGPUs: options.HostOptions.DisableGPU, + SkipUSBs: options.HostOptions.DisableUSB, + SkipCustomDevs: options.HostOptions.DisableCustomDevice, + EnableCudaHAMI: options.HostOptions.EnableCudaHAMI, + EnableCudaMps: options.HostOptions.EnableCudaMPS, + EnableContainerNPU: options.HostOptions.EnableContainerAscendNPU, + EnableWhitelist: options.HostOptions.EnableIsolatedDeviceWhitelist, + SriovNics: sriovNics, + OvsOffloadNics: offloadNics, + NvmePciDisks: options.HostOptions.PTNVMEConfigs, + AmdVgpuPFs: options.HostOptions.AMDVgpuPFs, + NvidiaVgpuPFs: options.HostOptions.NVIDIAVgpuPFs, + } + h.IsolatedDeviceMan.ProbePCIDevices(probeOpts) objs, err := h.getRemoteIsolatedDevices() if err != nil { @@ -2353,7 +2359,6 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { // detach device h.IsolatedDeviceMan.AppendDetachedDevice(&info) } - } h.IsolatedDeviceMan.StartDetachTask() @@ -2382,6 +2387,11 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { mtx.Lock() updateDevs.Add(obj) mtx.Unlock() + info := isolated_device.CloudDeviceInfo{} + if err := obj.Unmarshal(&info); err != nil { + return errors.Wrapf(err, "unmarshal isolated device %s to cloud device info", obj) + } + dev.SetDeviceInfo(info) } return nil } @@ -2649,9 +2659,17 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { hasVasmi := false hasNvidiasmi := false for _, dev := range devs { - devType := dev.GetDeviceType() - switch devType { - case string(isolated_device.ContainerDeviceTypeCphAMDGPU): + if !utils.IsInStringArray(dev.GetSharingMode(), api.VIRTUAL_SHARING_MODES) { + continue + } + if dev.GetDeviceType() == api.NETINT_TYPE { + hasNetint = true + continue + } + + vendorId := strings.Split(dev.GetVendorDeviceId(), ":")[0] + switch vendorId { + case api.AMD_VENDOR_ID: confMap, ok := conf[system_service.TELEGRAF_INPUT_RADEONTOP].(map[string]interface{}) if !ok { conf[system_service.TELEGRAF_INPUT_RADEONTOP] = map[string]interface{}{ @@ -2665,13 +2683,9 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) { confMap[system_service.TELEGRAF_INPUT_RADEONTOP_DEV_PATHS] = devPaths } } - case string(isolated_device.ContainerNetintCAQuadra), string(isolated_device.ContainerNetintCAASIC): - hasNetint = true - continue - case string(isolated_device.ContainerDeviceTypeVastaitechGpu): + case api.VASTAITECH_VENDOR_ID: hasVasmi = true - continue - case string(isolated_device.ContainerDeviceTypeNvidiaGpu), string(isolated_device.ContainerDeviceTypeNvidiaMps), string(isolated_device.ContainerDeviceTypeNvidiaGpuShare): + case api.NVIDIA_VENDOR_ID: hasNvidiasmi = true } } diff --git a/pkg/hostman/hostmetrics/container_metrics.go b/pkg/hostman/hostmetrics/container_metrics.go index 8af182678b..7021596705 100644 --- a/pkg/hostman/hostmetrics/container_metrics.go +++ b/pkg/hostman/hostmetrics/container_metrics.go @@ -146,7 +146,7 @@ func (m PodCphAmdGpuMetrics) GetUniformName() string { func (m PodCphAmdGpuMetrics) GetTag() map[string]string { return map[string]string{ "dev_id": m.DevId, - "dev_type": apis.CONTAINER_DEV_CPH_AMD_GPU, + "dev_type": apis.GPU_TYPE, } } @@ -183,7 +183,7 @@ func (m PodVastaitechGpuMetrics) GetUniformName() string { func (m PodVastaitechGpuMetrics) GetTag() map[string]string { return map[string]string{ "dev_id": m.DevId, - "dev_type": apis.CONTAINER_DEV_VASTAITECH_GPU, + "dev_type": apis.NPU_TYPE, } } @@ -226,14 +226,18 @@ func (m PodNvidiaGpuMetrics) GetUniformName() string { } func (m PodNvidiaGpuMetrics) GetTag() map[string]string { - devType := apis.CONTAINER_DEV_NVIDIA_GPU + devType := apis.GPU_TYPE + sharingMode := apis.DEVICE_SHARING_MODE_UNLIMITED if options.HostOptions.EnableCudaMPS { - devType = apis.CONTAINER_DEV_NVIDIA_MPS + sharingMode = apis.DEVICE_SHARING_MODE_MPS + } else if options.HostOptions.EnableCudaHAMI { + sharingMode = apis.DEVICE_SHARING_MODE_HAMI } return map[string]string{ "index": strconv.Itoa(m.Index), "physical_index": strconv.Itoa(m.PhysicalIndex), "dev_type": devType, + "sharing_mode": sharingMode, } } diff --git a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go index c879c468ef..d92f9568a8 100644 --- a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go +++ b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go @@ -76,7 +76,7 @@ func parseNvidiaGpuProcessMetrics(gpuMetricsStr string) []NvidiaGpuProcessMetric &processMetrics.Index, &processMetrics.Pid, &processMetrics.Type, &fb, &ccpm, &sm, &mem, &enc, &dec, &jpg, &ofa, &processMetrics.Command) if err != nil { - log.Errorf("failed parse nvidia gpu metrics %s: %s", line, err) + log.Debugf("failed parse nvidia gpu metrics %s: %s", line, err) continue } if processMetrics.Command == "nvidia-cuda-mps" || processMetrics.Command == "-" { @@ -159,10 +159,14 @@ func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string podDesc := pod.GetDesc() hasGpu := false for i := range podDesc.IsolatedDevices { - if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, compute.CONTAINER_GPU_TYPES) { - hasGpu = true - break + if podDesc.IsolatedDevices[i].DevType != compute.GPU_TYPE { + continue } + if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) { + continue + } + hasGpu = true + break } if !hasGpu { return true diff --git a/pkg/hostman/hostmetrics/hostmetrics.go b/pkg/hostman/hostmetrics/hostmetrics.go index 0f0c86c4cf..5d16131d12 100644 --- a/pkg/hostman/hostmetrics/hostmetrics.go +++ b/pkg/hostman/hostmetrics/hostmetrics.go @@ -661,12 +661,17 @@ func NewGuestPodMonitor( hasCphAmdGpu := false hasVastaitechGpu := false for i := range podDesc.IsolatedDevices { - if utils.IsInStringArray(podDesc.IsolatedDevices[i].DevType, compute.NVIDIA_GPU_TYPES) { + if !utils.IsInStringArray(podDesc.IsolatedDevices[i].SharingMode, compute.VIRTUAL_SHARING_MODES) { + continue + } + vendorId := strings.Split(podDesc.IsolatedDevices[i].VendorDeviceId, ":")[0] + switch vendorId { + case compute.NVIDIA_VENDOR_ID: hasNvGpu = true - } else if podDesc.IsolatedDevices[i].DevType == compute.CONTAINER_DEV_VASTAITECH_GPU { - hasVastaitechGpu = true - } else if podDesc.IsolatedDevices[i].DevType == compute.CONTAINER_DEV_CPH_AMD_GPU { + case compute.AMD_VENDOR_ID: hasCphAmdGpu = true + case compute.VASTAITECH_VENDOR_ID: + hasVastaitechGpu = true } } diff --git a/pkg/hostman/isolated_device/container_device.go b/pkg/hostman/isolated_device/container_device.go index 47d401e32b..f3eb5b49f3 100644 --- a/pkg/hostman/isolated_device/container_device.go +++ b/pkg/hostman/isolated_device/container_device.go @@ -44,23 +44,24 @@ const ( ContainerDeviceTypeNvidiaGpu ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU ContainerDeviceTypeNvidiaMps ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_MPS ContainerDeviceTypeNvidiaGpuShare ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_GPU_SHARE + ContainerDeviceTypeNvidiaHAMI ContainerDeviceType = api.CONTAINER_DEV_NVIDIA_HAMI ContainerDeviceTypeAscendNpu ContainerDeviceType = api.CONTAINER_DEV_ASCEND_NPU ContainerDeviceTypeVastaitechGpu ContainerDeviceType = api.CONTAINER_DEV_VASTAITECH_GPU ) -func GetContainerDeviceManager(t ContainerDeviceType) (IContainerDeviceManager, error) { - man, ok := containerDeviceManagers[t] +func GetContainerDeviceManager(devType ContainerDeviceType) (IContainerDeviceManager, error) { + man, ok := containerDeviceManagers[devType] if !ok { - return nil, errors.Wrapf(errors.ErrNotFound, "not found container device manager by %q", t) + return nil, errors.Wrapf(errors.ErrNotFound, "not found container device manager by %q", devType) } return man, nil } func RegisterContainerDeviceManager(man IContainerDeviceManager) { - if _, ok := containerDeviceManagers[man.GetType()]; ok { - panic(fmt.Sprintf("container device manager %s is already registered", man.GetType())) + if _, ok := containerDeviceManagers[man.GetRegisterType()]; ok { + panic(fmt.Sprintf("container device manager %s is already registered", man.GetRegisterType())) } - containerDeviceManagers[man.GetType()] = man + containerDeviceManagers[man.GetRegisterType()] = man } func GetContainerCDIManager(t apis.ContainerCDIKind) (IContainerCDIManager, error) { @@ -89,7 +90,7 @@ type ContainerDeviceConfiguration struct { } type IContainerDeviceManager interface { - GetType() ContainerDeviceType + GetRegisterType() ContainerDeviceType NewDevices(dev *ContainerDevice) ([]IDevice, error) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) ProbeDevices() ([]IDevice, error) diff --git a/pkg/hostman/isolated_device/container_device/ascend_npu.go b/pkg/hostman/isolated_device/container_device/ascend_npu.go index 867f006988..1b01af6415 100644 --- a/pkg/hostman/isolated_device/container_device/ascend_npu.go +++ b/pkg/hostman/isolated_device/container_device/ascend_npu.go @@ -24,7 +24,9 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -41,7 +43,9 @@ func (m *ascendNPUManager) GetContainerExtraConfigures(devs []*hostapi.Container if dev.IsolatedDevice == nil { continue } - if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeAscendNpu { + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*ascendNPUManager); !ok { continue } npus = append(npus, dev.IsolatedDevice.Path) @@ -104,7 +108,7 @@ func newAscendNPUManager() *ascendNPUManager { } func (m *ascendNPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { - return getAscendNpus() + return m.getAscendNpus() } func (m *ascendNPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { @@ -137,15 +141,29 @@ func (m *ascendNPUManager) NewContainerDevices(input *hostapi.ContainerCreateInp }, nil } -func (m *ascendNPUManager) GetType() isolated_device.ContainerDeviceType { +func (m *ascendNPUManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeAscendNpu } +func (m *ascendNPUManager) GetDevType() string { + return computeapi.NPU_TYPE +} + +func (m *ascendNPUManager) GetSharingMode() string { + return computeapi.DEVICE_SHARING_MODE_UNLIMITED +} + type ascnedNPU struct { + manager *ascendNPUManager + *BaseDevice } -func getAscendNpus() ([]isolated_device.IDevice, error) { +func (dev *ascnedNPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func (m *ascendNPUManager) getAscendNpus() ([]isolated_device.IDevice, error) { devs := make([]isolated_device.IDevice, 0) // Show all device's topology information out, err := procutils.NewRemoteCommandAsFarAsPossible("npu-smi", "info").Output() @@ -188,7 +206,8 @@ func getAscendNpus() ([]isolated_device.IDevice, error) { } dev := isolated_device.NewPCIDevice2(pciOutput[0]) npuDev := &ascnedNPU{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeAscendNpu, devPath), + manager: m, + BaseDevice: NewBaseDevice(dev, computeapi.NPU_TYPE, devPath, computeapi.DEVICE_SHARING_MODE_UNLIMITED, 1), } npuDev.SetModelName(npuName) diff --git a/pkg/hostman/isolated_device/container_device/base_dev.go b/pkg/hostman/isolated_device/container_device/base_dev.go index 00fabfaf5e..836d3384a7 100644 --- a/pkg/hostman/isolated_device/container_device/base_dev.go +++ b/pkg/hostman/isolated_device/container_device/base_dev.go @@ -29,13 +29,17 @@ import ( type BaseDevice struct { *isolated_device.SBaseDevice - Path string + Path string + VirtualNum int } -func NewBaseDevice(dev *isolated_device.PCIDevice, devType isolated_device.ContainerDeviceType, devPath string) *BaseDevice { +func NewBaseDevice( + dev *isolated_device.PCIDevice, devType, devPath, sharingMode string, virtualNum int, +) *BaseDevice { return &BaseDevice{ - SBaseDevice: isolated_device.NewBaseDevice(dev, string(devType)), + SBaseDevice: isolated_device.NewBaseDevice(dev, string(devType), sharingMode), Path: devPath, + VirtualNum: virtualNum, } } @@ -59,6 +63,14 @@ func (c BaseDevice) GetDevicePath() string { return c.Path } +func (c BaseDevice) GetVirtualNum() int { + return c.VirtualNum +} + +func (c BaseDevice) HotPluggable() bool { + return false +} + func (c *BaseDevice) SetDevicePath(devPath string) { c.Path = devPath } @@ -116,7 +128,7 @@ func getGPUPCIAddr(linkPartName string) (string, error) { return fullAddr, nil } -func NewPCIGPURenderBaseDevice(devPath string, index int, devType isolated_device.ContainerDeviceType) (*BaseDevice, error) { +func NewPCIGPURenderBaseDevice(devPath string, virtualNum int, devType, sharingMode string) (*BaseDevice, error) { dir := "/dev/dri/by-path/" entries, err := os.ReadDir(dir) if err != nil { @@ -144,9 +156,9 @@ func NewPCIGPURenderBaseDevice(devPath string, index int, devType isolated_devic return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", pciAddr) } dev := isolated_device.NewPCIDevice2(pciOutput[0]) - devAddr := dev.Addr - baseDev := NewBaseDevice(dev, devType, devPath) - baseDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, index), devAddr) + // devAddr := dev.Addr + baseDev := NewBaseDevice(dev, devType, devPath, sharingMode, virtualNum) + //baseDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, index), devAddr) return baseDev, nil } diff --git a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go index 6dcb4136cc..0521d9c4f6 100644 --- a/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go +++ b/pkg/hostman/isolated_device/container_device/cph_amd_gpu.go @@ -18,6 +18,7 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" @@ -39,10 +40,18 @@ func (m *cphAMDGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { return nil, nil } -func (m *cphAMDGPUManager) GetType() isolated_device.ContainerDeviceType { +func (m *cphAMDGPUManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeCphAMDGPU } +func (m *cphAMDGPUManager) GetDevType() string { + return compute.GPU_TYPE +} + +func (m *cphAMDGPUManager) GetSharingMode() string { + return compute.DEVICE_SHARING_MODE_UNLIMITED +} + func (m *cphAMDGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { if !strings.HasPrefix(dev.Path, "/dev/dri/renderD") { return nil, errors.Errorf("device path %q doesn't start with /dev/dri/renderD", dev.Path) @@ -50,15 +59,11 @@ func (m *cphAMDGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]i if err := CheckVirtualNumber(dev); err != nil { return nil, err } - gpuDevs := make([]isolated_device.IDevice, 0) - for i := 0; i < dev.VirtualNumber; i++ { - gpuDev, err := newCphAMDGPU(dev.Path, i) - if err != nil { - return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i) - } - gpuDevs = append(gpuDevs, gpuDev) + gpuDev, err := m.newCphAMDGPU(dev.Path, compute.DEVICE_SHARING_MODE_UNLIMITED, dev.VirtualNumber) + if err != nil { + return nil, errors.Wrapf(err, "new CPH AMD GPU with virtual num %d", dev.VirtualNumber) } - return gpuDevs, nil + return []isolated_device.IDevice{gpuDev}, nil } func (m *cphAMDGPUManager) getDeviceHostPathByAddr(dev *hostapi.ContainerDevice) (string, error) { @@ -83,15 +88,22 @@ func (m *cphAMDGPUManager) GetContainerExtraConfigures(devs []*hostapi.Container } type cphAMDGPU struct { + manager *cphAMDGPUManager + *BaseDevice } -func newCphAMDGPU(devPath string, index int) (*cphAMDGPU, error) { - dev, err := NewPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeCphAMDGPU) +func (dev *cphAMDGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func (m *cphAMDGPUManager) newCphAMDGPU(devPath, sharingMode string, virtualNum int) (*cphAMDGPU, error) { + dev, err := NewPCIGPURenderBaseDevice(devPath, virtualNum, compute.GPU_TYPE, sharingMode) if err != nil { return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice") } return &cphAMDGPU{ + manager: m, BaseDevice: dev, }, nil } diff --git a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go index 37e1a5e2db..f532a81e45 100644 --- a/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go +++ b/pkg/hostman/isolated_device/container_device/cph_aosp_binder.go @@ -20,6 +20,7 @@ import ( "path/filepath" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" @@ -43,16 +44,25 @@ func init() { type cphAOSPBinderManager struct { controlDevicePath string controlName string + initialized bool } func newCphAOSPBinderManager() *cphAOSPBinderManager { return &cphAOSPBinderManager{} } -func (m *cphAOSPBinderManager) GetType() isolated_device.ContainerDeviceType { +func (m *cphAOSPBinderManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeCphASOPBinder } +func (m *cphAOSPBinderManager) GetDevType() string { + return compute.BINDER_TYPE +} + +func (m *cphAOSPBinderManager) GetSharingMode() string { + return compute.DEVICE_SHARING_MODE_UNLIMITED +} + func (m *cphAOSPBinderManager) ProbeDevices() ([]isolated_device.IDevice, error) { return nil, nil } @@ -64,26 +74,27 @@ func (m *cphAOSPBinderManager) NewDevices(dev *isolated_device.ContainerDevice) if err := m.initialize(dev); err != nil { return nil, errors.Wrap(err, "initialize") } - devs := make([]isolated_device.IDevice, 0) - for i := 0; i < dev.VirtualNumber; i++ { - newDev, err := m.newDeviceByIndex(i) - if err != nil { - return nil, errors.Wrapf(err, "new device by index %d", i) - } - devs = append(devs, newDev) - } - return devs, nil -} -func (m *cphAOSPBinderManager) newDeviceByIndex(index int) (isolated_device.IDevice, error) { - dev, err := newCphAOSPBinder(index, m.controlDevicePath) - if err != nil { - return nil, errors.Wrap(err, "newCphAOSPBinder") + id := "aosp_binder" + ndev := &isolated_device.PCIDevice{ + Addr: m.controlName, + VendorId: CPH_AOSP_VENDOR_ID, + DeviceId: CPH_AOSP_DEVICE_ID, + ModelName: CPH_AOSP_BINDER_MODEL_NAME, } - return dev, nil + devPath := fmt.Sprintf("/dev/%s", id) + binderDev := &cphAOSPBinder{ + manager: m, + BaseDevice: NewBaseDevice(ndev, compute.BINDER_TYPE, devPath, compute.DEVICE_SHARING_MODE_UNLIMITED, dev.VirtualNumber), + ControlPath: m.controlDevicePath, + } + return []isolated_device.IDevice{binderDev}, nil } func (m *cphAOSPBinderManager) initialize(dev *isolated_device.ContainerDevice) error { + if m.initialized { + return errors.Errorf("cphAOSPBinderManager already initialized") + } ctrlPath := CPH_AOSP_BINDER_CONTROL_DEV_PATH info, err := os.Stat(ctrlPath) if err != nil { @@ -91,6 +102,7 @@ func (m *cphAOSPBinderManager) initialize(dev *isolated_device.ContainerDevice) } m.controlDevicePath = ctrlPath m.controlName = info.Name() + m.initialized = true return nil } @@ -159,22 +171,11 @@ func (m *cphAOSPBinderManager) ensureBinderDevice(ctrName string, dev *hostapi.C } type cphAOSPBinder struct { + manager *cphAOSPBinderManager *BaseDevice ControlPath string } -func newCphAOSPBinder(idx int, ctrPath string) (*cphAOSPBinder, error) { - id := fmt.Sprintf("aosp_binder_%d", idx) - dev := &isolated_device.PCIDevice{ - Addr: fmt.Sprintf("%d", idx), - VendorId: CPH_AOSP_VENDOR_ID, - DeviceId: CPH_AOSP_DEVICE_ID, - ModelName: CPH_AOSP_BINDER_MODEL_NAME, - } - devPath := fmt.Sprintf("/dev/%s", id) - binderDev := &cphAOSPBinder{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeCphASOPBinder, devPath), - ControlPath: ctrPath, - } - return binderDev, nil +func (dev *cphAOSPBinder) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager } diff --git a/pkg/hostman/isolated_device/container_device/net_int_device.go b/pkg/hostman/isolated_device/container_device/net_int_device.go index 33e8b21ff8..e021ac9a87 100644 --- a/pkg/hostman/isolated_device/container_device/net_int_device.go +++ b/pkg/hostman/isolated_device/container_device/net_int_device.go @@ -21,6 +21,7 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -93,7 +94,7 @@ func newNetintDeviceManager(devType isolated_device.ContainerDeviceType, reg *re } } -func (m *netintDeviceManager) GetType() isolated_device.ContainerDeviceType { +func (m *netintDeviceManager) GetRegisterType() isolated_device.ContainerDeviceType { return m.devType } @@ -159,35 +160,27 @@ func (m *netintDeviceManager) NewDevices(dev *isolated_device.ContainerDevice) ( } result := make([]isolated_device.IDevice, 0) for _, nvmeDev := range nvmeDevs { - for i := 0; i < dev.VirtualNumber; i++ { - newDev, err := m.newDeviceByIndex(nvmeDev, i) - if err != nil { - return nil, errors.Wrapf(err, "newDeviceByIndex %#v %d", nvmeDev, i) - } - result = append(result, newDev) + devIdx, err := nvmeDev.GetIndex() + if err != nil { + return nil, err } + devInfo := &isolated_device.PCIDevice{ + Addr: strconv.Itoa(devIdx), + VendorId: NETINT_VENDOR_ID, + DeviceId: NETINT_DEVICE_ID, + ModelName: nvmeDev.ModelNumber, + } + newDev := &netintDevice{ + manager: m, + BaseDevice: NewBaseDevice(devInfo, compute.NETINT_TYPE, nvmeDev.DevicePath, compute.DEVICE_SHARING_MODE_UNLIMITED, dev.VirtualNumber), + info: nvmeDev, + } + + result = append(result, newDev) } return result, nil } -func (m *netintDeviceManager) newDeviceByIndex(dev *NetintDeviceInfo, idx int) (*netintDevice, error) { - devIdx, err := dev.GetIndex() - if err != nil { - return nil, errors.Wrap(err, "dev.GetIndex") - } - devInfo := &isolated_device.PCIDevice{ - Addr: fmt.Sprintf("%d-%d", devIdx, idx), - VendorId: NETINT_VENDOR_ID, - DeviceId: NETINT_DEVICE_ID, - ModelName: dev.ModelNumber, - } - nvmeDev := &netintDevice{ - BaseDevice: NewBaseDevice(devInfo, m.devType, dev.DevicePath), - info: dev, - } - return nvmeDev, nil -} - func (m *netintDeviceManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { dev := input.IsolatedDevice if !fileutils2.Exists(dev.Path) { @@ -217,6 +210,7 @@ func (m *netintDeviceManager) GetContainerExtraConfigures(devs []*hostapi.Contai } type netintDevice struct { + manager *netintDeviceManager *BaseDevice info *NetintDeviceInfo } @@ -224,3 +218,7 @@ type netintDevice struct { func (d netintDevice) GetNVMESizeMB() int { return d.info.PhysicalSize / 1024 / 1024 } + +func (dev *netintDevice) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} diff --git a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go index 7e95f0d7a1..b1f433f5f1 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_gpu.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_gpu.go @@ -23,9 +23,10 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/sets" + api "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -40,12 +41,12 @@ func newNvidiaGPUManager() *nvidiaGPUManager { return &nvidiaGPUManager{} } -func (m *nvidiaGPUManager) GetType() isolated_device.ContainerDeviceType { +func (m *nvidiaGPUManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeNvidiaGpu } func (m *nvidiaGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { - return probeNvidiaGpus() + return probeNvidiaGpus(api.DEVICE_SHARING_MODE_EXCLUSIVE, m) } func (m *nvidiaGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { @@ -62,13 +63,12 @@ func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.Container if dev.IsolatedDevice == nil { continue } - types := sets.NewString( - string(isolated_device.ContainerDeviceTypeNvidiaGpu), - string(isolated_device.ContainerDeviceTypeNvidiaGpuShare), - ) - if !types.Has(dev.IsolatedDevice.DeviceType) { + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*nvidiaGPUManager); !ok { continue } + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) } if len(gpuIds) == 0 { @@ -91,6 +91,7 @@ func (m *nvidiaGPUManager) GetContainerExtraConfigures(devs []*hostapi.Container } type nvidiaGPU struct { + manager isolated_device.IContainerDeviceManager *BaseDevice memSize int @@ -114,20 +115,26 @@ func (dev *nvidiaGPU) GetDeviceMinor() int { return dev.deviceMinor } -func probeNvidiaGpus() ([]isolated_device.IDevice, error) { +func (dev *nvidiaGPU) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + +func probeNvidiaGpus(sharingMode string, manager isolated_device.IContainerDeviceManager) ([]isolated_device.IDevice, error) { if nvidiaGpuUsages != nil { res := make([]isolated_device.IDevice, 0) for pciAddr, dev := range nvidiaGpuUsages { if dev.Used { continue } + dev := nvidiaGpuUsages[pciAddr].nvidiaGPU + dev.manager = manager res = append(res, nvidiaGpuUsages[pciAddr].nvidiaGPU) } nvidiaGpuUsages = nil return res, nil } - devs, err := getNvidiaGPUs() + devs, err := getNvidiaGPUs(sharingMode, manager) if err != nil { return nil, err } @@ -138,7 +145,7 @@ func probeNvidiaGpus() ([]isolated_device.IDevice, error) { return res, nil } -func getNvidiaGPUs() ([]*nvidiaGPU, error) { +func getNvidiaGPUs(sharingMode string, manager isolated_device.IContainerDeviceManager) ([]*nvidiaGPU, error) { devs := make([]*nvidiaGPU, 0) // nvidia-smi --query-gpu=gpu_uuid,gpu_name,gpu_bus_id --format=csv // uuid, name, pci.bus_id @@ -190,7 +197,8 @@ func getNvidiaGPUs() ([]*nvidiaGPU, error) { } gpuDev := &nvidiaGPU{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaGpu, gpuId), + manager: manager, + BaseDevice: NewBaseDevice(dev, api.GPU_TYPE, gpuId, sharingMode, 1), memSize: memSize, gpuIndex: indexInt, deviceMinor: driverInfo.DeviceMinor, diff --git a/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go b/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go index faf0a90690..84885baa1d 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_gpu_share.go @@ -20,6 +20,10 @@ import ( "path/filepath" "strings" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + api "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/hostman/isolated_device" @@ -30,14 +34,50 @@ func init() { } type nvidiaGPUShareManager struct { - nvidiaGPUManager +} + +func (m *nvidiaGPUShareManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + return nil, nil, nil +} + +func (m *nvidiaGPUShareManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + gpuIds := []string{} + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*nvidiaGPUShareManager); !ok { + continue + } + + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) + } + if len(gpuIds) == 0 { + return nil, nil + } + retEnvs := []*runtimeapi.KeyValue{} + if len(gpuIds) > 0 { + retEnvs = append(retEnvs, []*runtimeapi.KeyValue{ + { + Key: "NVIDIA_VISIBLE_DEVICES", + Value: strings.Join(gpuIds, ","), + }, + { + Key: "NVIDIA_DRIVER_CAPABILITIES", + Value: "all", + }, + }...) + } + return retEnvs, nil } func newNvidiaGPUShareManager() *nvidiaGPUShareManager { return &nvidiaGPUShareManager{} } -func (m *nvidiaGPUShareManager) GetType() isolated_device.ContainerDeviceType { +func (m *nvidiaGPUShareManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeNvidiaGpuShare } @@ -53,19 +93,16 @@ func (m *nvidiaGPUShareManager) NewDevices(dev *isolated_device.ContainerDevice) return nil, err } - gpuDevs := make([]isolated_device.IDevice, 0) - for i := 0; i < dev.VirtualNumber; i++ { - gpuDev, err := newNvidiaGpuShare(dev.Path, i) - if err != nil { - return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i) - } - gpuDevs = append(gpuDevs, gpuDev) + gpuDev, err := m.newNvidiaGpuShare(dev.Path, dev.VirtualNumber) + if err != nil { + return nil, errors.Wrap(err, "new CPH AMD GPU") } - return gpuDevs, nil + return []isolated_device.IDevice{gpuDev}, nil } type nvidiaGpuShareDev struct { nvidiaGPU + manager *nvidiaGPUShareManager CardPath string RenderPath string @@ -79,6 +116,10 @@ func (dev *nvidiaGpuShareDev) GetRenderPath() string { return dev.RenderPath } +func (dev *nvidiaGpuShareDev) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + type nvidiaGpuUsage struct { *nvidiaGPU @@ -91,7 +132,7 @@ func getNvidiaGpuUsage() (map[string]*nvidiaGpuUsage, error) { if nvidiaGpuUsages != nil { return nvidiaGpuUsages, nil } - devs, err := getNvidiaGPUs() + devs, err := getNvidiaGPUs(api.DEVICE_SHARING_MODE_UNLIMITED, nil) if err != nil { return nil, err } @@ -109,15 +150,15 @@ func getNvidiaGpuUsage() (map[string]*nvidiaGpuUsage, error) { return nvidiaGpuUsages, nil } -func newNvidiaGpuShare(devPath string, index int) (*nvidiaGpuShareDev, error) { +func (m *nvidiaGPUShareManager) newNvidiaGpuShare(devPath string, virtualNumber int) (*nvidiaGpuShareDev, error) { devUsages, err := getNvidiaGpuUsage() if err != nil { return nil, errors.Wrap(err, "getNvidiaGpuUsage") } - dev, err := NewPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeNvidiaGpuShare) + dev, err := NewPCIGPURenderBaseDevice(devPath, virtualNumber, api.GPU_TYPE, api.DEVICE_SHARING_MODE_UNLIMITED) if err != nil { - return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice") + return nil, errors.Wrap(err, "NewPCIGPURenderBaseDevice") } devAddr := dev.GetOriginAddr() cardPath := path.Join("/dev/dri/by-path", fmt.Sprintf("pci-0000:%s-card", devAddr)) @@ -133,6 +174,7 @@ func newNvidiaGpuShare(devPath string, index int) (*nvidiaGpuShareDev, error) { dev.SetDevicePath(nvidiaGPUDev.Path) return &nvidiaGpuShareDev{ + manager: m, nvidiaGPU: nvidiaGPU{ BaseDevice: dev, memSize: devUsages[devAddr].memSize, diff --git a/pkg/hostman/isolated_device/container_device/nvidia_hami.go b/pkg/hostman/isolated_device/container_device/nvidia_hami.go new file mode 100644 index 0000000000..debf5d75e8 --- /dev/null +++ b/pkg/hostman/isolated_device/container_device/nvidia_hami.go @@ -0,0 +1,116 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container_device + +import ( + "fmt" + "strings" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" + "yunion.io/x/onecloud/pkg/hostman/options" +) + +func init() { + isolated_device.RegisterContainerDeviceManager(newNvidiaHAMIManager()) +} + +type nvidiaHAMIManager struct { +} + +func (m *nvidiaHAMIManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { + return nil, nil +} + +func (m *nvidiaHAMIManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { + return nil, nil, nil +} + +func newNvidiaHAMIManager() *nvidiaHAMIManager { + return &nvidiaHAMIManager{} +} + +func (m *nvidiaHAMIManager) GetRegisterType() isolated_device.ContainerDeviceType { + return isolated_device.ContainerDeviceTypeNvidiaHAMI +} + +func (m *nvidiaHAMIManager) ProbeDevices() ([]isolated_device.IDevice, error) { + return probeNvidiaGpus(computeapi.DEVICE_SHARING_MODE_HAMI, m) +} + +func (m *nvidiaHAMIManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { + gpuIds := []string{} + memoryLimit := "" + smLimit := "" + for _, dev := range devs { + if dev.IsolatedDevice == nil { + continue + } + + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*nvidiaHAMIManager); !ok { + continue + } + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) + if memoryLimit == "" { + memoryLimit = fmt.Sprintf("%dM", dev.IsolatedDevice.MemoryLimit) + } + if smLimit == "" && dev.IsolatedDevice.SmUtilLimit > 0 { + smLimit = fmt.Sprintf("%d", dev.IsolatedDevice.SmUtilLimit) + } + } + if len(gpuIds) == 0 { + return nil, nil + } + retEnvs := []*runtimeapi.KeyValue{} + if len(gpuIds) > 0 { + retEnvs = append(retEnvs, []*runtimeapi.KeyValue{ + { + Key: "NVIDIA_VISIBLE_DEVICES", + Value: strings.Join(gpuIds, ","), + }, + { + Key: "NVIDIA_DRIVER_CAPABILITIES", + Value: "all", + }, + { + Key: "LD_PRELOAD", + Value: options.HostOptions.HAMICoreLibvgpuPath, + }, + { + Key: "CUDA_DEVICE_MEMORY_LIMIT", + Value: memoryLimit, + }, + }...) + if len(smLimit) > 0 { + retEnvs = append(retEnvs, &runtimeapi.KeyValue{ + Key: "CUDA_DEVICE_SM_LIMIT", + Value: smLimit, + }) + } + } + return retEnvs, []*runtimeapi.Mount{ + { + ContainerPath: options.HostOptions.HAMICoreLibvgpuPath, + HostPath: options.HostOptions.HAMICoreLibvgpuPath, + Readonly: true, + }, + } +} diff --git a/pkg/hostman/isolated_device/container_device/nvidia_mps.go b/pkg/hostman/isolated_device/container_device/nvidia_mps.go index b79ee26188..29dd7c5230 100644 --- a/pkg/hostman/isolated_device/container_device/nvidia_mps.go +++ b/pkg/hostman/isolated_device/container_device/nvidia_mps.go @@ -15,7 +15,6 @@ package container_device import ( - "fmt" "strconv" "strings" @@ -24,7 +23,9 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/util/procutils" @@ -44,12 +45,12 @@ func newNvidiaMPSManager() *nvidiaMPSManager { return &nvidiaMPSManager{} } -func (m *nvidiaMPSManager) GetType() isolated_device.ContainerDeviceType { +func (m *nvidiaMPSManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeNvidiaMps } func (m *nvidiaMPSManager) ProbeDevices() ([]isolated_device.IDevice, error) { - return getNvidiaMPSGpus(options.HostOptions.CudaMPSReplicas) + return m.getNvidiaMPSGpus(options.HostOptions.CudaMPSReplicas) } func (m *nvidiaMPSManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { @@ -60,7 +61,7 @@ func (m *nvidiaMPSManager) NewDevices(dev *isolated_device.ContainerDevice) ([]i return nil, err } - return getNvidiaMPSGpusByDevPath(dev.VirtualNumber, dev.Path) + return m.getNvidiaMPSGpusByDevPath(dev.VirtualNumber, dev.Path) } func (m *nvidiaMPSManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { @@ -81,9 +82,12 @@ func (m *nvidiaMPSManager) GetContainerExtraConfigures(devs []*hostapi.Container if dev.IsolatedDevice == nil { continue } - if isolated_device.ContainerDeviceType(dev.IsolatedDevice.DeviceType) != isolated_device.ContainerDeviceTypeNvidiaMps { + iDev := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByCloudId(dev.IsolatedDevice.Id) + devMan := iDev.GetContainerDeviceManager() + if _, ok := devMan.(*nvidiaMPSManager); !ok { continue } + gpuIds = append(gpuIds, dev.IsolatedDevice.Path) } if len(gpuIds) == 0 { @@ -116,6 +120,7 @@ func (m *nvidiaMPSManager) GetContainerExtraConfigures(devs []*hostapi.Container } type nvidiaMPS struct { + manager *nvidiaMPSManager *BaseDevice MemSizeMB int @@ -145,6 +150,10 @@ func (c *nvidiaMPS) GetNvidiaMpsThreadPercentage() int { return c.ThreadPercentage } +func (dev *nvidiaMPS) GetContainerDeviceManager() isolated_device.IContainerDeviceManager { + return dev.manager +} + func parseMemSize(memTotalStr string) (int, error) { if !strings.HasSuffix(memTotalStr, " MiB") { return -1, errors.Errorf("unknown mem string suffix") @@ -153,9 +162,9 @@ func parseMemSize(memTotalStr string) (int, error) { return strconv.Atoi(memStr) } -func getNvidiaMPSGpusByDevPath(cudaMPSReplicas int, devPath string) ([]isolated_device.IDevice, error) { +func (m *nvidiaMPSManager) getNvidiaMPSGpusByDevPath(cudaMPSReplicas int, devPath string) ([]isolated_device.IDevice, error) { configuredByDevPath = true - pDev, err := NewPCIGPURenderBaseDevice(devPath, 0, isolated_device.ContainerDeviceTypeNvidiaMps) + pDev, err := NewPCIGPURenderBaseDevice(devPath, 0, computeapi.GPU_TYPE, computeapi.DEVICE_SHARING_MODE_MPS) if err != nil { return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice") } @@ -200,21 +209,17 @@ func getNvidiaMPSGpusByDevPath(cudaMPSReplicas int, devPath string) ([]isolated_ if err != nil { return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr) } - - for i := 0; i < cudaMPSReplicas; i++ { - dev := isolated_device.NewPCIDevice2(pciOutput[0]) - gpuDev := &nvidiaMPS{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaMps, gpuId), - MemSizeMB: memSize / cudaMPSReplicas, - MemTotalMB: memSize, - ThreadPercentage: 100 / cudaMPSReplicas, - gpuIndex: index, - } - gpuDev.SetModelName(gpuName) - devAddr := gpuDev.GetAddr() - gpuDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, i), devAddr) - devs = append(devs, gpuDev) + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + gpuDev := &nvidiaMPS{ + manager: m, + BaseDevice: NewBaseDevice(dev, computeapi.GPU_TYPE, gpuId, computeapi.DEVICE_SHARING_MODE_MPS, cudaMPSReplicas), + MemSizeMB: memSize / cudaMPSReplicas, + MemTotalMB: memSize, + ThreadPercentage: 100 / cudaMPSReplicas, + gpuIndex: index, } + gpuDev.SetModelName(gpuName) + devs = append(devs, gpuDev) } if len(devs) == 0 { return nil, nil @@ -222,7 +227,7 @@ func getNvidiaMPSGpusByDevPath(cudaMPSReplicas int, devPath string) ([]isolated_ return devs, nil } -func getNvidiaMPSGpus(cudaMPSReplicas int) ([]isolated_device.IDevice, error) { +func (m *nvidiaMPSManager) getNvidiaMPSGpus(cudaMPSReplicas int) ([]isolated_device.IDevice, error) { if configuredByDevPath { return nil, nil } @@ -257,20 +262,17 @@ func getNvidiaMPSGpus(cudaMPSReplicas int) ([]isolated_device.IDevice, error) { if err != nil { return nil, errors.Wrapf(err, "GetPCIStrByAddr %s", gpuPciAddr) } - for i := 0; i < cudaMPSReplicas; i++ { - dev := isolated_device.NewPCIDevice2(pciOutput[0]) - gpuDev := &nvidiaMPS{ - BaseDevice: NewBaseDevice(dev, isolated_device.ContainerDeviceTypeNvidiaMps, gpuId), - MemSizeMB: memSize / cudaMPSReplicas, - MemTotalMB: memSize, - ThreadPercentage: 100 / cudaMPSReplicas, - gpuIndex: index, - } - gpuDev.SetModelName(gpuName) - devAddr := gpuDev.GetAddr() - gpuDev.SetAddr(fmt.Sprintf("%s-%d", devAddr, i), devAddr) - devs = append(devs, gpuDev) + dev := isolated_device.NewPCIDevice2(pciOutput[0]) + gpuDev := &nvidiaMPS{ + manager: m, + BaseDevice: NewBaseDevice(dev, computeapi.GPU_TYPE, gpuId, computeapi.DEVICE_SHARING_MODE_MPS, cudaMPSReplicas), + MemSizeMB: memSize / cudaMPSReplicas, + MemTotalMB: memSize, + ThreadPercentage: 100 / cudaMPSReplicas, + gpuIndex: index, } + gpuDev.SetModelName(gpuName) + devs = append(devs, gpuDev) } if len(devs) == 0 { return nil, nil diff --git a/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go b/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go index c8ef70ad1c..e4a7d69f09 100644 --- a/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go +++ b/pkg/hostman/isolated_device/container_device/vastaitech_gpu.go @@ -22,6 +22,7 @@ import ( "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/pkg/errors" @@ -40,7 +41,7 @@ func newVastaitechGPUManager() isolated_device.IContainerDeviceManager { return &vastaitechGPUManager{} } -func (v vastaitechGPUManager) GetType() isolated_device.ContainerDeviceType { +func (v vastaitechGPUManager) GetRegisterType() isolated_device.ContainerDeviceType { return isolated_device.ContainerDeviceTypeVastaitechGpu } @@ -56,7 +57,7 @@ var vastaitechRelatedDevices = map[string]string{ VASTAITECH_VACC: "/dev/vacc%d", } -func (v vastaitechGPUManager) getRelatedDevices(index int) map[string]string { +func (v *vastaitechGPUManager) getRelatedDevices(index int) map[string]string { devs := make(map[string]string) for key, devFmt := range vastaitechRelatedDevices { devs[key] = fmt.Sprintf(devFmt, index) @@ -64,12 +65,12 @@ func (v vastaitechGPUManager) getRelatedDevices(index int) map[string]string { return devs } -func (v vastaitechGPUManager) getDriRenderPrefix() string { +func (v *vastaitechGPUManager) getDriRenderPrefix() string { return "/dev/dri/renderD" } // getVastaitechDriStartIndexFromByPath 扫描 /dev/dri/by-path/，找到名称含 va_card 的 -render 链接对应的最小 renderD 编号 -func (v vastaitechGPUManager) getVastaitechDriStartIndexFromByPath() (int, error) { +func (v *vastaitechGPUManager) getVastaitechDriStartIndexFromByPath() (int, error) { const byPathDir = "/dev/dri/by-path" entries, err := os.ReadDir(byPathDir) if err != nil { @@ -107,11 +108,11 @@ func (v vastaitechGPUManager) getVastaitechDriStartIndexFromByPath() (int, error return *minIdx, nil } -func (v vastaitechGPUManager) getDriStartIndex() (int, error) { +func (v *vastaitechGPUManager) getDriStartIndex() (int, error) { return v.getVastaitechDriStartIndexFromByPath() } -func (v vastaitechGPUManager) getRelatedDeviceStartIndex(driPath string) (int, error) { +func (v *vastaitechGPUManager) getRelatedDeviceStartIndex(driPath string) (int, error) { prefix := v.getDriRenderPrefix() if !strings.HasPrefix(driPath, prefix) { return -1, errors.Errorf("device path %q doesn't start with /dev/dri/renderD", driPath) @@ -132,7 +133,7 @@ func (v vastaitechGPUManager) getRelatedDeviceStartIndex(driPath string) (int, e return idx, nil } -func (v vastaitechGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { +func (v *vastaitechGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { idx, err := v.getRelatedDeviceStartIndex(dev.Path) if err != nil { return nil, errors.Wrap(err, "get related device start index") @@ -146,18 +147,15 @@ func (v vastaitechGPUManager) NewDevices(dev *isolated_device.ContainerDevice) ( if err := CheckVirtualNumber(dev); err != nil { return nil, err } - gpuDevs := make([]isolated_device.IDevice, 0) - for i := 0; i < dev.VirtualNumber; i++ { - gpuDev, err := newVastaitechGPU(dev.Path, i) - if err != nil { - return nil, errors.Wrapf(err, "new CPH AMD GPU with index %d", i) - } - gpuDevs = append(gpuDevs, gpuDev) + gpuDev, err := v.newVastaitechGPU(dev.Path, dev.VirtualNumber) + if err != nil { + return nil, errors.Wrap(err, "new CPH AMD GPU") } - return gpuDevs, nil + + return []isolated_device.IDevice{gpuDev}, nil } -func (v vastaitechGPUManager) getCommonDevices() []*runtimeapi.Device { +func (v *vastaitechGPUManager) getCommonDevices() []*runtimeapi.Device { vatools := "/dev/vatools" vaSync := "/dev/va_sync" devs := []*runtimeapi.Device{} @@ -171,7 +169,7 @@ func (v vastaitechGPUManager) getCommonDevices() []*runtimeapi.Device { return devs } -func (v vastaitechGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { +func (v *vastaitechGPUManager) NewContainerDevices(input *hostapi.ContainerCreateInput, dev *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { driHostPath := dev.IsolatedDevice.Path idx, err := v.getRelatedDeviceStartIndex(driHostPath) if err != nil { @@ -195,22 +193,26 @@ func (v vastaitechGPUManager) NewContainerDevices(input *hostapi.ContainerCreate return devs, v.getCommonDevices(), nil } -func (v vastaitechGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { +func (v *vastaitechGPUManager) ProbeDevices() ([]isolated_device.IDevice, error) { return nil, nil } -func (v vastaitechGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { +func (v *vastaitechGPUManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { return nil, nil } type vastaitechGPU struct { *BaseDevice + manager *vastaitechGPUManager } -func newVastaitechGPU(devPath string, index int) (*vastaitechGPU, error) { - dev, err := NewPCIGPURenderBaseDevice(devPath, index, isolated_device.ContainerDeviceTypeVastaitechGpu) +func (v *vastaitechGPUManager) newVastaitechGPU(devPath string, virtualNum int) (*vastaitechGPU, error) { + dev, err := NewPCIGPURenderBaseDevice(devPath, virtualNum, computeapi.GPU_TYPE, computeapi.DEVICE_SHARING_MODE_UNLIMITED) if err != nil { return nil, errors.Wrap(err, "new PCIGPURenderBaseDevice") } - return &vastaitechGPU{BaseDevice: dev}, nil + return &vastaitechGPU{ + BaseDevice: dev, + manager: v, + }, nil } diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index af84be1be3..2daca609dd 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -254,9 +254,9 @@ type sGPUBaseDevice struct { *SBaseDevice } -func newGPUBaseDevice(dev *PCIDevice, devType string) *sGPUBaseDevice { +func newGPUBaseDevice(dev *PCIDevice, devType, sharingMode string) *sGPUBaseDevice { return &sGPUBaseDevice{ - SBaseDevice: NewBaseDevice(dev, devType), + SBaseDevice: NewBaseDevice(dev, devType, sharingMode), } } @@ -332,9 +332,9 @@ type sGPUHPCDevice struct { *sGPUBaseDevice } -func NewGPUHPCDevice(dev *PCIDevice) *sGPUHPCDevice { +func NewGPUHPCDevice(dev *PCIDevice, sharingMode string) *sGPUHPCDevice { gpuDev := &sGPUHPCDevice{ - sGPUBaseDevice: newGPUBaseDevice(dev, api.GPU_HPC_TYPE), + sGPUBaseDevice: newGPUBaseDevice(dev, api.GPU_TYPE, api.DEVICE_SHARING_MODE_EXCLUSIVE), } return gpuDev } diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 74e096f598..2fd73b7010 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" @@ -62,6 +63,9 @@ type CloudDeviceInfo struct { MpsThreadPercentage int `json:"mps_thread_percentage"` NumaNode int `json:"numa_node"` PcieInfo *api.IsolatedDevicePCIEInfo `json:"pcie_info"` + VirtualNum int `json:"virtual_num"` + HotPluggable bool `json:"hot_pluggable"` + SharingMode string `json:"sharing_mode"` // The frame rate limiter (FRL) configuration in frames per second FRL string `json:"frl"` @@ -110,10 +114,14 @@ type IDevice interface { GetVendorDeviceId() string GetAddr() string GetDeviceType() string + GetSharingMode() string GetModelName() string CustomProbe(idx int) error SetDeviceInfo(info CloudDeviceInfo) DetectByAddr() error + GetVirtualNum() int + GetContainerDeviceManager() IContainerDeviceManager + HotPluggable() bool GetPassthroughOptions() map[string]string GetPassthroughCmd(index int) string @@ -161,11 +169,12 @@ type IsolatedDeviceManager interface { GetDevices() []IDevice GetDeviceByIdent(vendorDevId, addr, mdevId string) IDevice GetDeviceByAddr(addr string) IDevice - ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableCudaMps, enableContainerNPU, enableWhitelist bool) + GetDeviceByCloudId(cloudId string) IDevice + ProbePCIDevices(opts *SIsolatedDeviceProbeOptions) StartDetachTask() BatchCustomProbe() AppendDetachedDevice(dev *CloudDeviceInfo) - GetQemuParams(devAddrs []string) *QemuParams + //GetQemuParams(devAddrs []string) *QemuParams CheckDevIsNeedUpdate(dev IDevice, devInfo *CloudDeviceInfo) bool } @@ -233,10 +242,12 @@ func (man *isolatedDeviceManager) probeContainerDevices() { } } -func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaMps bool) { +func (man *isolatedDeviceManager) probeContainerNvidiaGPUs(enableCudaHAMI, enableCudaMps bool) { devType := ContainerDeviceTypeNvidiaGpu if enableCudaMps { devType = ContainerDeviceTypeNvidiaMps + } else if enableCudaHAMI { + devType = ContainerDeviceTypeNvidiaHAMI } devman, err := GetContainerDeviceManager(devType) @@ -301,7 +312,7 @@ func (man *isolatedDeviceManager) probeGPUS(skipGPUs bool, amdVgpuPFs, nvidiaVgp } } for idx, gpu := range gpus { - man.devices = append(man.devices, NewGPUHPCDevice(gpu)) + man.devices = append(man.devices, NewGPUHPCDevice(gpu, api.DEVICE_SHARING_MODE_EXCLUSIVE)) log.Infof("Add GPU device: %d => %#v", idx, gpu) } } @@ -441,12 +452,27 @@ func (man *isolatedDeviceManager) probeNVIDIAVgpus(nvidiaVgpuPFs []string) { } } -func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks, amdVgpuPFs, nvidiaVgpuPFs []string, enableCudaMps, enableContainerNPU, enableWhitelist bool) { +type SIsolatedDeviceProbeOptions struct { + SkipGPUs bool + SkipUSBs bool + SkipCustomDevs bool + + EnableCudaHAMI bool + EnableCudaMps bool + EnableContainerNPU bool + EnableWhitelist bool + + SriovNics, OvsOffloadNics []HostNic + + NvmePciDisks, AmdVgpuPFs, NvidiaVgpuPFs []string +} + +func (man *isolatedDeviceManager) ProbePCIDevices(opts *SIsolatedDeviceProbeOptions) { man.devices = make([]IDevice, 0) if man.host.IsContainerHost() { man.probeContainerDevices() - man.probeContainerNvidiaGPUs(enableCudaMps) - man.probeContainerAscendNPUs(enableContainerNPU) + man.probeContainerNvidiaGPUs(opts.EnableCudaHAMI, opts.EnableCudaMps) + man.probeContainerAscendNPUs(opts.EnableContainerNPU) } else { devModels, err := man.getCustomIsolatedDeviceModels() if err != nil { @@ -454,21 +480,22 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustom man.host.AppendError(fmt.Sprintf("get custom isolated device devModels %s", err.Error()), "isolated_devices", "", "") return } - man.probeUSBs(skipUSBs) - man.probeCustomPCIDevs(skipCustomDevs, devModels, GpuClassCodes) - man.probeSRIOVNics(sriovNics) - man.probeOffloadNICS(ovsOffloadNics) - man.probeAMDVgpus(amdVgpuPFs) - man.probeNVIDIAVgpus(nvidiaVgpuPFs) - man.probeGPUS(skipGPUs, amdVgpuPFs, nvidiaVgpuPFs, enableWhitelist, devModels) + man.probeUSBs(opts.SkipUSBs) + man.probeCustomPCIDevs(opts.SkipCustomDevs, devModels, GpuClassCodes) + man.probeSRIOVNics(opts.SriovNics) + man.probeOffloadNICS(opts.OvsOffloadNics) + man.probeAMDVgpus(opts.AmdVgpuPFs) + man.probeNVIDIAVgpus(opts.NvidiaVgpuPFs) + man.probeGPUS(opts.SkipGPUs, opts.AmdVgpuPFs, opts.NvidiaVgpuPFs, opts.EnableWhitelist, devModels) } } type IsolatedDeviceModel struct { - DevType string `json:"dev_type"` - VendorId string `json:"vendor_id"` - DeviceId string `json:"device_id"` - Model string `json:"model"` + DevType string `json:"dev_type"` + VendorId string `json:"vendor_id"` + DeviceId string `json:"device_id"` + Model string `json:"model"` + HotPluggable tristate.TriState `json:"hot_pluggable"` } func (man *isolatedDeviceManager) getCustomIsolatedDeviceModels() ([]IsolatedDeviceModel, error) { @@ -571,6 +598,15 @@ func (man *isolatedDeviceManager) CheckDevIsNeedUpdate(dev IDevice, devInfo *Clo return true } } + if dev.GetVirtualNum() != devInfo.VirtualNum { + return true + } + if dev.HotPluggable() != devInfo.HotPluggable { + return true + } + if dev.GetSharingMode() != devInfo.SharingMode { + return true + } return false } @@ -616,6 +652,15 @@ func (man *isolatedDeviceManager) GetDeviceByAddr(addr string) IDevice { return nil } +func (man *isolatedDeviceManager) GetDeviceByCloudId(cloudId string) IDevice { + for _, dev := range man.devices { + if dev.GetCloudId() == cloudId { + return dev + } + } + return nil +} + func (man *isolatedDeviceManager) BatchCustomProbe() { for i, dev := range man.devices { if err := dev.CustomProbe(i); err != nil { @@ -661,10 +706,6 @@ func (man *isolatedDeviceManager) StartDetachTask() { }() } -func (man *isolatedDeviceManager) GetQemuParams(devAddrs []string) *QemuParams { - return getQemuParams(man, devAddrs) -} - type SBaseDevice struct { dev *PCIDevice originAddr string @@ -672,13 +713,15 @@ type SBaseDevice struct { hostId string guestId string devType string + sharingMode string detectedOnHost bool } -func NewBaseDevice(dev *PCIDevice, devType string) *SBaseDevice { +func NewBaseDevice(dev *PCIDevice, devType, sharingMode string) *SBaseDevice { return &SBaseDevice{ - dev: dev, - devType: devType, + dev: dev, + devType: devType, + sharingMode: sharingMode, } } @@ -763,6 +806,10 @@ func (dev *SBaseDevice) GetDeviceType() string { return dev.devType } +func (dev *SBaseDevice) GetSharingMode() string { + return dev.sharingMode +} + func (dev *SBaseDevice) GetPfName() string { return "" } @@ -822,6 +869,18 @@ func (dev *SBaseDevice) GetGuestId() string { return dev.guestId } +func (dev *SBaseDevice) GetVirtualNum() int { + return 1 +} + +func (dev *SBaseDevice) HotPluggable() bool { + return true +} + +func (dev *SBaseDevice) GetContainerDeviceManager() IContainerDeviceManager { + return nil +} + func (dev *SBaseDevice) GetNvidiaMpsMemoryLimit() int { return -1 } @@ -856,6 +915,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { "addr": dev.GetAddr(), "model": dev.GetModelName(), "vendor_device_id": dev.GetVendorDeviceId(), + "virtual_num": dev.GetVirtualNum(), + "hot_pluggable": dev.HotPluggable(), + "sharing_mode": dev.GetSharingMode(), } detected := false if err := dev.DetectByAddr(); err == nil { @@ -868,9 +930,9 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { if len(dev.GetHostId()) != 0 { data["host_id"] = dev.GetHostId() } - if len(dev.GetGuestId()) != 0 { - data["guest_id"] = dev.GetGuestId() - } + //if len(dev.GetGuestId()) != 0 { + // data["guest_id"] = dev.GetGuestId() + //} if len(dev.GetWireId()) != 0 { data["wire_id"] = dev.GetWireId() } @@ -891,7 +953,6 @@ func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { } else { log.Debugf("failed get dev %s numa node %s", dev.GetAddr(), err) } - if dev.GetMdevId() != "" { data["mdev_id"] = dev.GetMdevId() } @@ -1083,52 +1144,3 @@ func bashOutput(cmd string) ([]string, error) { func bashRawOutput(cmd string) ([]string, error) { return bashCmdOutput(cmd, false) } - -type QemuParams struct { - Cpu string - Vga string - Devices []string -} - -func getQemuParams(man *isolatedDeviceManager, devAddrs []string) *QemuParams { - if len(devAddrs) == 0 { - return nil - } - devCmds := []string{} - cpuCmd := DEFAULT_CPU_CMD - vgaCmd := DEFAULT_VGA_CMD - // group by device type firstly - devices := make(map[string][]IDevice, 0) - for _, addr := range devAddrs { - dev := man.GetDeviceByAddr(addr) - if dev == nil { - log.Warningf("IsolatedDeviceManager not found dev %#v, ignore it!", addr) - continue - } - devType := dev.GetDeviceType() - if _, ok := devices[devType]; !ok { - devices[devType] = []IDevice{dev} - } else { - devices[devType] = append(devices[devType], dev) - } - } - - for devType, devs := range devices { - log.Debugf("get devices %s command", devType) - for idx, dev := range devs { - devCmds = append(devCmds, getDeviceCmd(dev, idx)) - if dev.GetVGACmd() != vgaCmd && dev.GetDeviceType() == api.GPU_VGA_TYPE { - vgaCmd = dev.GetVGACmd() - } - if dev.GetCPUCmd() != cpuCmd { - cpuCmd = dev.GetCPUCmd() - } - } - } - - return &QemuParams{ - Cpu: cpuCmd, - Vga: vgaCmd, - Devices: devCmds, - } -} diff --git a/pkg/hostman/isolated_device/nic.go b/pkg/hostman/isolated_device/nic.go index 6c76d80a43..c066190e47 100644 --- a/pkg/hostman/isolated_device/nic.go +++ b/pkg/hostman/isolated_device/nic.go @@ -100,7 +100,7 @@ func getSRIOVNics(hostNics []HostNic) ([]*sSRIOVNicDevice, error) { func NewSRIOVNicDevice(dev *PCIDevice, devType, wireId, pfName string, virtfn int, isInfinibandNic bool) *sSRIOVNicDevice { return &sSRIOVNicDevice{ - sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType), + sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType, api.DEVICE_SHARING_MODE_SRIOV), WireId: wireId, pfName: pfName, virtfn: virtfn, diff --git a/pkg/hostman/isolated_device/nvidia_vgpu.go b/pkg/hostman/isolated_device/nvidia_vgpu.go index ebadf887fb..54ee46bbfc 100644 --- a/pkg/hostman/isolated_device/nvidia_vgpu.go +++ b/pkg/hostman/isolated_device/nvidia_vgpu.go @@ -30,11 +30,12 @@ import ( ) type sNVIDIAVgpuDevice struct { - pfDev *PCIDevice - cloudId string - hostId string - guestId string - devType string + pfDev *PCIDevice + cloudId string + hostId string + guestId string + devType string + sharingMode string mdevId string model string @@ -85,6 +86,10 @@ func (dev *sNVIDIAVgpuDevice) GetDeviceType() string { return dev.devType } +func (dev *sNVIDIAVgpuDevice) GetSharingMode() string { + return dev.sharingMode +} + func (dev *sNVIDIAVgpuDevice) GetModelName() string { modelName := dev.pfDev.ModelName if dev.pfDev.ModelName == "" { @@ -101,6 +106,18 @@ func (dev *sNVIDIAVgpuDevice) GetDevicePath() string { return "" } +func (dev *sNVIDIAVgpuDevice) GetVirtualNum() int { + return 1 +} + +func (dev *sNVIDIAVgpuDevice) HotPluggable() bool { + return true +} + +func (dev *sNVIDIAVgpuDevice) GetContainerDeviceManager() IContainerDeviceManager { + return nil +} + func (dev *sNVIDIAVgpuDevice) GetNvidiaMpsMemoryLimit() int { return -1 } @@ -265,13 +282,14 @@ func (dev *sNVIDIAVgpuDevice) GetPCIEInfo() *compute.IsolatedDevicePCIEInfo { return dev.pfDev.PCIEInfo } -func NewNvidiaVgpuDevice(dev *PCIDevice, devType, mdevId, model string, profile map[string]string) *sNVIDIAVgpuDevice { +func NewNvidiaVgpuDevice(dev *PCIDevice, devType, sharingMode, mdevId, model string, profile map[string]string) *sNVIDIAVgpuDevice { return &sNVIDIAVgpuDevice{ - pfDev: dev, - devType: devType, - mdevId: mdevId, - model: model, - profile: profile, + pfDev: dev, + devType: devType, + sharingMode: sharingMode, + mdevId: mdevId, + model: model, + profile: profile, } } @@ -318,7 +336,7 @@ func getNvidiaVGpus(gpuPF string) ([]*sNVIDIAVgpuDevice, error) { profile[key] = strings.TrimSpace(value) } } - mdev := NewNvidiaVgpuDevice(pfDev, compute.LEGACY_VGPU_TYPE, files[i].Name(), model, profile) + mdev := NewNvidiaVgpuDevice(pfDev, compute.GPU_TYPE, compute.DEVICE_SHARING_MODE_MDEV, files[i].Name(), model, profile) nvidiaVgpus = append(nvidiaVgpus, mdev) } return nvidiaVgpus, nil diff --git a/pkg/hostman/isolated_device/nvme.go b/pkg/hostman/isolated_device/nvme.go index df183e3a42..1bb5d25c22 100644 --- a/pkg/hostman/isolated_device/nvme.go +++ b/pkg/hostman/isolated_device/nvme.go @@ -48,7 +48,7 @@ func (dev *sNVMEDevice) GetNVMESizeMB() int { func newNVMEDevice(dev *PCIDevice, devType string, sizeMB int) *sNVMEDevice { return &sNVMEDevice{ - SBaseDevice: NewBaseDevice(dev, devType), + SBaseDevice: NewBaseDevice(dev, devType, api.DEVICE_SHARING_MODE_EXCLUSIVE), sizeMB: sizeMB, } } diff --git a/pkg/hostman/isolated_device/pci_device.go b/pkg/hostman/isolated_device/pci_device.go index 3f9091c18c..c6db74da17 100644 --- a/pkg/hostman/isolated_device/pci_device.go +++ b/pkg/hostman/isolated_device/pci_device.go @@ -21,10 +21,14 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" ) type sGeneralPCIDevice struct { *SBaseDevice + + hotPluggable bool } func (dev *sGeneralPCIDevice) GetVGACmd() string { @@ -39,9 +43,14 @@ func (dev *sGeneralPCIDevice) GetQemuId() string { return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_")) } -func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice { +func (dev *sGeneralPCIDevice) HotPluggable() bool { + return dev.hotPluggable +} + +func newGeneralPCIDevice(dev *PCIDevice, devType string, hotPluggable bool) *sGeneralPCIDevice { return &sGeneralPCIDevice{ - SBaseDevice: NewBaseDevice(dev, devType), + SBaseDevice: NewBaseDevice(dev, devType, computeapi.DEVICE_SHARING_MODE_EXCLUSIVE), + hotPluggable: hotPluggable, } } @@ -71,7 +80,7 @@ func getPassthroughPCIDevs(devModel IsolatedDeviceModel, filteredCodes []string) errs = append(errs, errors.Wrapf(err, "get dev %s iommu group devices by model: %s", dev.Addr, jsonutils.Marshal(devModel))) continue } - devs = append(devs, newGeneralPCIDevice(dev, devModel.DevType)) + devs = append(devs, newGeneralPCIDevice(dev, devModel.DevType, devModel.HotPluggable.Bool())) } return devs, errors.NewAggregate(errs) } diff --git a/pkg/hostman/isolated_device/sriov_base.go b/pkg/hostman/isolated_device/sriov_base.go index b473fa0b31..ba80c829c6 100644 --- a/pkg/hostman/isolated_device/sriov_base.go +++ b/pkg/hostman/isolated_device/sriov_base.go @@ -70,9 +70,9 @@ func detectSRIOVDevice(vfBDF string) (*PCIDevice, error) { return dev, nil } -func newSRIOVBaseDevice(dev *PCIDevice, devType string) *sSRIOVBaseDevice { +func newSRIOVBaseDevice(dev *PCIDevice, devType, sharingMode string) *sSRIOVBaseDevice { return &sSRIOVBaseDevice{ - SBaseDevice: NewBaseDevice(dev, devType), + SBaseDevice: NewBaseDevice(dev, devType, sharingMode), } } diff --git a/pkg/hostman/isolated_device/sriov_vgpu.go b/pkg/hostman/isolated_device/sriov_vgpu.go index bfad55c88f..14f8310fff 100644 --- a/pkg/hostman/isolated_device/sriov_vgpu.go +++ b/pkg/hostman/isolated_device/sriov_vgpu.go @@ -32,9 +32,9 @@ type sSRIOVGpuDevice struct { *sSRIOVBaseDevice } -func NewSRIOVGpuDevice(dev *PCIDevice, devType string) *sSRIOVGpuDevice { +func NewSRIOVGpuDevice(dev *PCIDevice, devType, sharingMode string) *sSRIOVGpuDevice { return &sSRIOVGpuDevice{ - sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType), + sSRIOVBaseDevice: newSRIOVBaseDevice(dev, devType, sharingMode), } } @@ -71,7 +71,7 @@ func getSRIOVGpus(gpuPF string) ([]*sSRIOVGpuDevice, error) { if err != nil { return nil, err } - sriovGPUs = append(sriovGPUs, NewSRIOVGpuDevice(vfDev, compute.SRIOV_VGPU_TYPE)) + sriovGPUs = append(sriovGPUs, NewSRIOVGpuDevice(vfDev, compute.GPU_TYPE, compute.DEVICE_SHARING_MODE_SRIOV)) } } return sriovGPUs, err diff --git a/pkg/hostman/isolated_device/usb.go b/pkg/hostman/isolated_device/usb.go index 0730b14eae..ae25860d95 100644 --- a/pkg/hostman/isolated_device/usb.go +++ b/pkg/hostman/isolated_device/usb.go @@ -39,7 +39,7 @@ type sUSBDevice struct { // TODO: rename PCIDevice func newUSBDevice(dev *PCIDevice, lsusbLine *sLsusbLine) *sUSBDevice { return &sUSBDevice{ - SBaseDevice: NewBaseDevice(dev, api.USB_TYPE), + SBaseDevice: NewBaseDevice(dev, api.USB_TYPE, api.DEVICE_SHARING_MODE_EXCLUSIVE), lsusbLine: lsusbLine, } } diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 1f4592ba37..d88223c078 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -269,9 +269,12 @@ type SHostOptions struct { CudaMPSLogDirectory string `help:"cuda mps log dir" default:"/tmp/nvidia-mps/log"` CudaMPSReplicas int `help:"cuda mps replicas" default:"10"` + EnableCudaHAMI bool `help:"enable cuda hami" default:"true"` + HAMICoreLibvgpuPath string `help:"hami core libvgpu.so path" default:"/opt/cloud/hami/libvgpu.so"` + SkipCheckKernelMods []string `help:"skip check kernel modules"` - EnableContainerAscendNPU bool `help:"enable container npu" default:"false"` + EnableContainerAscendNPU bool `help:"enable container npu" default:"true"` EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` diff --git a/pkg/hostman/storageman/container_storage/local_raw.go b/pkg/hostman/storageman/container_storage/local_raw.go index b31de94ae7..0b93b25bd8 100644 --- a/pkg/hostman/storageman/container_storage/local_raw.go +++ b/pkg/hostman/storageman/container_storage/local_raw.go @@ -33,12 +33,20 @@ func init() { type localLoopDiskManager struct { } -func (l localLoopDiskManager) GetType() isolated_device.ContainerDeviceType { +func (l localLoopDiskManager) GetRegisterType() isolated_device.ContainerDeviceType { return api.CONTAINER_STORAGE_LOCAL_RAW } +func (m *localLoopDiskManager) GetDevType() string { + return api.GPU_TYPE +} + +func (m *localLoopDiskManager) GetSharingMode() string { + return api.DEVICE_SHARING_MODE_UNLIMITED +} + func (l localLoopDiskManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { - return nil, errors.Errorf("%s storage doesn't support NewDevices", l.GetType()) + return nil, errors.Errorf("%s storage doesn't support NewDevices", l.GetRegisterType()) } func (l localLoopDiskManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { diff --git a/pkg/llm/models/sku.go b/pkg/llm/models/sku.go index 48af415309..27b6014656 100644 --- a/pkg/llm/models/sku.go +++ b/pkg/llm/models/sku.go @@ -87,7 +87,7 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred if input.Devices != nil { for i := range *input.Devices { if (*input.Devices)[i].DevType == "" { - (*input.Devices)[i].DevType = computeapi.CONTAINER_DEV_NVIDIA_GPU_SHARE + (*input.Devices)[i].DevType = computeapi.GPU_TYPE } } } diff --git a/pkg/mcclient/modules/compute/mod_isolatedevices.go b/pkg/mcclient/modules/compute/mod_isolatedevices.go index 3efc8e4a21..0073c7b6d5 100644 --- a/pkg/mcclient/modules/compute/mod_isolatedevices.go +++ b/pkg/mcclient/modules/compute/mod_isolatedevices.go @@ -20,15 +20,27 @@ import ( ) var ( - IsolatedDevices modulebase.ResourceManager + IsolatedDevices modulebase.ResourceManager + ServerIsolatedDevices modulebase.JointResourceManager ) func init() { IsolatedDevices = modules.NewComputeManager("isolated_device", "isolated_devices", - []string{"ID", "Dev_type", + []string{"ID", "Dev_type", "Sharing_mode", "Model", "Addr", "Vendor_device_id", "Mdev_id", "Host_id", "Host", "numa_node", "Guest_id", "Guest", "Guest_status", "Device_path", "Render_path", "PCIE_Info", "Index", "Device_minor"}, []string{}) modules.RegisterCompute(&IsolatedDevices) + + ServerIsolatedDevices = modules.NewJointComputeManager( + "guestisolateddevice", + "guestisolateddevices", + []string{"Guest_ID", "Guest", "Isolated_device_ID", "Index", "Dev_type", "Sharing_mode", + "Device_memory_size", "Sm_util_limit", "Network_index", "Disk_index"}, + []string{}, + &Servers, + &IsolatedDevices, + ) + modules.RegisterCompute(&ServerIsolatedDevices) } diff --git a/pkg/mcclient/options/compute/isolated_devices.go b/pkg/mcclient/options/compute/isolated_devices.go index 76d6836823..aaf127c3c8 100644 --- a/pkg/mcclient/options/compute/isolated_devices.go +++ b/pkg/mcclient/options/compute/isolated_devices.go @@ -43,6 +43,35 @@ func (o *DeviceListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(o) } +type ServerDeviceListOptions struct { + DeviceListOptions + IsolateDeviceIds []string `json:"isolate_device_ids"` +} + +func (o *ServerDeviceListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type DeviceCreateOptions struct { + options.BaseCreateOptions + + HostId string + DevType string + SharingMode string + Model string + HotPluggable bool + Addr string + DevicePath string + VendorDeviceId string + MemorySize int + VirtualNum int + NumaNode int +} + +func (o *DeviceCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + type DeviceShowOptions struct { options.BaseIdOptions } diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index cf24607d0f..b8f8361559 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -81,6 +81,7 @@ type ServerListOptions struct { WithUserMeta *bool `help:"filter by user metadata" negative:"without_user_meta"` + IsolateDeviceId string `help:"filter guest with isolated device id" json:"isolate_device_id"` WithHost *bool `help:"filter guest with host or not" negative:"without_host"` SnapshotpolicyId string `help:"filter guest with snapshotpolicy or not" json:"snapshotpolicy_id"` BindingDisksSnapshotpolicy *bool `help:"filter guest with disks binding snapshotpolicy or not" negative:"no-binding-disks-snapshotpolicy" json:"binding_disks_snapshotpolicy"` diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index fb341f8db1..8012b86ab2 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -17,6 +17,7 @@ package predicates import ( "context" "fmt" + "path" "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/core" @@ -61,36 +62,35 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit, return false, nil } -func (f *IsolatedDevicePredicate) getIsolatedDeviceCountByType(getter core.CandidatePropertyGetter, devType string) int { - devs := getter.UnusedIsolatedDevicesByType(devType) - if devType != compute.CONTAINER_DEV_NVIDIA_MPS && devType != compute.CONTAINER_DEV_NVIDIA_GPU_SHARE { - return len(devs) - } else { - devMap := map[string]struct{}{} - for _, dev := range devs { - devMap[dev.DevicePath] = struct{}{} +func (f *IsolatedDevicePredicate) getIsolatedDeviceCountBySharingMode(sharingMode string, devs []*core.IsolatedDeviceDesc) int { + if sharingMode == compute.DEVICE_SHARING_MODE_HAMI { + ret := 0 + for i := range devs { + ret += devs[i].AvailableMemorySize() } - return len(devMap) + return ret } + ret := 0 + for i := range devs { + ret += devs[i].AvailableNum() + } + return ret } -// countDevicesWithMinMemory counts free devices of the given dev_type whose -// MemorySize satisfies the minimum requirement. Devices with MemorySize == 0 -// are treated as "unknown" and pass through (so newly-introduced rows that -// haven't been backfilled yet don't accidentally exclude every host). -// For NVIDIA_MPS / NVIDIA_GPU_SHARE the count is deduplicated by DevicePath, -// matching getIsolatedDeviceCountByType. -func (f *IsolatedDevicePredicate) countDevicesWithMinMemory(getter core.CandidatePropertyGetter, devType string, minMemoryMb int) int { - devs := getter.UnusedIsolatedDevicesByType(devType) - isShared := devType == compute.CONTAINER_DEV_NVIDIA_MPS || devType == compute.CONTAINER_DEV_NVIDIA_GPU_SHARE - return countDevicesWithMinMemoryFromList(devs, isShared, minMemoryMb) +// countDevicesWithMinMemory counts available capacity for devices of the given +// dev_type whose MemorySize satisfies the minimum requirement. Devices with +// MemorySize == 0 are treated as "unknown" and pass through for non-HAMI modes +// so rows that have not been backfilled yet do not exclude every host. +func (f *IsolatedDevicePredicate) countDevicesWithMinMemory(getter core.CandidatePropertyGetter, devType, sharingMode string, minMemoryMb int) int { + devs := getter.AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode) + return countDevicesWithMinMemoryFromList(devs, sharingMode, minMemoryMb) } // countDevicesWithMinMemoryFromList is the pure-function core of the memory // fit count, factored out for unit testing. Callers pass an already-filtered // list (typically by dev_type). -func countDevicesWithMinMemoryFromList(devs []*core.IsolatedDeviceDesc, isShared bool, minMemoryMb int) int { - if !isShared { +func countDevicesWithMinMemoryFromList(devs []*core.IsolatedDeviceDesc, sharingMode string, minMemoryMb int) int { + if sharingMode != compute.DEVICE_SHARING_MODE_HAMI { n := 0 for _, d := range devs { if d.MemorySize > 0 && d.MemorySize < minMemoryMb { @@ -100,14 +100,43 @@ func countDevicesWithMinMemoryFromList(devs []*core.IsolatedDeviceDesc, isShared } return n } - seen := map[string]struct{}{} + n := 0 for _, d := range devs { - if d.MemorySize > 0 && d.MemorySize < minMemoryMb { + if d.AvailableMemorySize() < minMemoryMb { continue } - seen[d.DevicePath] = struct{}{} + n++ } - return len(seen) + return n +} + +func filterDevicesByTypeSharingMode(devs []*core.IsolatedDeviceDesc, devType, sharingMode string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range devs { + if devType != "" && devType != dev.DevType { + continue + } + if sharingMode != "" && dev.SharingMode != sharingMode { + continue + } + ret = append(ret, dev) + } + return ret +} + +func isolatedDeviceRequestAmount(dev *compute.IsolatedDeviceConfig) int { + if dev.SharingMode == compute.DEVICE_SHARING_MODE_HAMI { + return dev.MemoryRequest + } + return 1 +} + +func isolatedDeviceMinMemory(dev *compute.IsolatedDeviceConfig) int { + minMemMb := dev.MemoryMb + if dev.SharingMode == compute.DEVICE_SHARING_MODE_HAMI && dev.MemoryRequest > minMemMb { + minMemMb = dev.MemoryRequest + } + return minMemMb } func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { @@ -131,6 +160,7 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c getter := c.Getter() minCapacity := int64(0xFFFFFFFF) + pendingUsage := getter.GetPendingUsage().IsolatedDevice // check by specify device id for _, dev := range reqIsoDevs { @@ -138,8 +168,8 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c continue } if fDev := getter.GetIsolatedDevice(dev.Id); fDev != nil { - if len(fDev.GuestID) != 0 { - h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.Id, fDev.GuestID)) + if fDev.IsUsedUp() { + h.Exclude(fmt.Sprintf("IsolatedDevice %q already used up", dev.Id)) return h.GetResult() } } else { @@ -148,30 +178,29 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } minCapacity = 1 } - - reqCount := len(reqIsoDevs) - freeCount := len(getter.UnusedIsolatedDevices()) - getter.GetPendingUsage().IsolatedDevice - totalCount := len(getter.GetIsolatedDevices()) - - // check host isolated device count - if freeCount < reqCount { - h.AppendInsufficientResourceError(int64(reqCount), int64(totalCount), int64(freeCount)) - h.Exclude(fmt.Sprintf( - "IsolatedDevice count not enough, request: %d, hostTotal: %d, hostFree: %d", - reqCount, totalCount, freeCount)) - return h.GetResult() + type reqKey struct { + devType string + sharingMode string } - // check host device by type - devTypeRequest := make(map[string]int, 0) + devTypeRequest := make(map[reqKey]int, 0) for _, dev := range reqIsoDevs { if len(dev.DevType) != 0 { - devTypeRequest[dev.DevType] += 1 + key := reqKey{devType: dev.DevType, sharingMode: dev.SharingMode} + reqAmount := isolatedDeviceRequestAmount(dev) + if reqAmount <= 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice type %q sharing_mode %q request amount must be positive", dev.DevType, dev.SharingMode)) + return h.GetResult() + } + devTypeRequest[key] += reqAmount } } - for devType, reqCount := range devTypeRequest { - freeCount := f.getIsolatedDeviceCountByType(getter, devType) - if freeCount < reqCount { + for key, reqCount := range devTypeRequest { + devType, sharingMode := key.devType, key.sharingMode + devs := getter.AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode) + pendingCnt := pendingUsage.Get(path.Join(devType, sharingMode)) + freeCount := f.getIsolatedDeviceCountBySharingMode(sharingMode, devs) + if freeCount < (reqCount + pendingCnt) { h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount)) return h.GetResult() } @@ -182,16 +211,37 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } // check host device by model - devVendorModelRequest := make(map[string]int, 0) + type modelReqKey struct { + vendorModel string + devType string + sharingMode string + } + devVendorModelRequest := make(map[modelReqKey]int, 0) for _, dev := range reqIsoDevs { if len(dev.Model) != 0 { - devVendorModelRequest[fmt.Sprintf("%s:%s", dev.Vendor, dev.Model)] += 1 + key := modelReqKey{ + vendorModel: fmt.Sprintf("%s:%s", dev.Vendor, dev.Model), + devType: dev.DevType, + sharingMode: dev.SharingMode, + } + reqAmount := isolatedDeviceRequestAmount(dev) + if reqAmount <= 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q request amount must be positive", key.vendorModel)) + return h.GetResult() + } + devVendorModelRequest[key] += reqAmount } } - for vendorModel, reqCount := range devVendorModelRequest { - freeCount := len(getter.UnusedIsolatedDevicesByVendorModel(vendorModel)) - if freeCount < reqCount { - h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: %d", vendorModel, reqCount, freeCount)) + for key, reqCount := range devVendorModelRequest { + devs := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByVendorModel(key.vendorModel), key.devType, key.sharingMode) + if len(devs) == 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: 0", key.vendorModel, reqCount)) + return h.GetResult() + } + pendingCnt := pendingUsage.Get(path.Join(key.devType, key.sharingMode)) + freeCount := f.getIsolatedDeviceCountBySharingMode(key.sharingMode, devs) + if freeCount < (reqCount + pendingCnt) { + h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: %d", key.vendorModel, reqCount, freeCount)) return h.GetResult() } cap := freeCount / reqCount @@ -205,18 +255,20 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c // vram_claim_mb is honoured. Devices with memory_size == 0 are passed // through as unknown (see countDevicesWithMinMemory). type vramReqKey struct { - devType string - minMemMb int + devType string + sharingMode string + minMemMb int } vramReq := make(map[vramReqKey]int) for _, dev := range reqIsoDevs { - if dev.MemoryMb <= 0 { + minMemMb := isolatedDeviceMinMemory(dev) + if minMemMb <= 0 { continue } - vramReq[vramReqKey{dev.DevType, dev.MemoryMb}]++ + vramReq[vramReqKey{dev.DevType, dev.SharingMode, minMemMb}]++ } for k, reqCnt := range vramReq { - fit := f.countDevicesWithMinMemory(getter, k.devType, k.minMemMb) + fit := f.countDevicesWithMinMemory(getter, k.devType, k.sharingMode, k.minMemMb) if fit < reqCnt { h.Exclude(fmt.Sprintf( "IsolatedDevice type %q with memory >= %d MiB not enough, request: %d, hostFree: %d", @@ -230,16 +282,37 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } // check host device by device_path - devicePathReq := make(map[string]int, 0) + type devicePathReqKey struct { + devicePath string + devType string + sharingMode string + } + devicePathReq := make(map[devicePathReqKey]int, 0) for _, dev := range reqIsoDevs { if len(dev.DevicePath) != 0 { - devicePathReq[dev.DevicePath] += 1 + key := devicePathReqKey{ + devicePath: dev.DevicePath, + devType: dev.DevType, + sharingMode: dev.SharingMode, + } + reqAmount := isolatedDeviceRequestAmount(dev) + if reqAmount <= 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q request amount must be positive", dev.DevicePath)) + return h.GetResult() + } + devicePathReq[key] += reqAmount } } - for devPath, reqCnt := range devicePathReq { - freeCount := len(getter.UnusedIsolatedDevicesByDevicePath(devPath)) - if freeCount < reqCount { - h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q not enough, request: %d, hostFree: %d", devPath, reqCount, freeCount)) + for key, reqCnt := range devicePathReq { + devs := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByDevicePath(key.devicePath), key.devType, key.sharingMode) + if len(devs) == 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q not enough, request: %d, hostFree: 0", key.devicePath, reqCnt)) + return h.GetResult() + } + pendingCnt := pendingUsage.Get(path.Join(key.devType, key.sharingMode)) + freeCount := f.getIsolatedDeviceCountBySharingMode(key.sharingMode, devs) + if freeCount < (reqCnt + pendingCnt) { + h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q not enough, request: %d, hostFree: %d", key.devicePath, reqCnt, freeCount)) return h.GetResult() } cap := freeCount / reqCnt diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go index 2cc85907ef..fb49044dd0 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go @@ -17,20 +17,36 @@ package predicates import ( "testing" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/core" ) func TestCountDevicesWithMinMemoryFromList(t *testing.T) { mk := func(path string, memMb int) *core.IsolatedDeviceDesc { - return &core.IsolatedDeviceDesc{DevicePath: path, MemorySize: memMb} + return &core.IsolatedDeviceDesc{DevicePath: path, MemorySize: memMb, VirtualNum: 1} + } + mkVirtual := func(path string, memMb, virtualNum, allocated int) *core.IsolatedDeviceDesc { + return &core.IsolatedDeviceDesc{ + DevicePath: path, + MemorySize: memMb, + VirtualNum: virtualNum, + VirtualNumAllocated: allocated, + } + } + mkHAMI := func(path string, memMb, allocatedMb int) *core.IsolatedDeviceDesc { + return &core.IsolatedDeviceDesc{ + DevicePath: path, + MemorySize: memMb, + MemorySizeAllocated: allocatedMb, + } } cases := []struct { - name string - devs []*core.IsolatedDeviceDesc - shared bool - minMemMb int - want int + name string + devs []*core.IsolatedDeviceDesc + sharingMode string + minMemMb int + want int }{ { name: "plain GPU: 3 cards 24/40/80 GiB, request 30 GiB → 2 fit", @@ -39,7 +55,7 @@ func TestCountDevicesWithMinMemoryFromList(t *testing.T) { mk("/dev/nvidia1", 40960), mk("/dev/nvidia2", 81920), }, - shared: false, minMemMb: 30000, want: 2, + sharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, minMemMb: 30000, want: 2, }, { name: "plain GPU: request 0 (unconstrained) → all pass through", @@ -47,7 +63,7 @@ func TestCountDevicesWithMinMemoryFromList(t *testing.T) { mk("/dev/nvidia0", 24576), mk("/dev/nvidia1", 40960), }, - shared: false, minMemMb: 0, want: 2, + sharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, minMemMb: 0, want: 2, }, { name: "unknown MemorySize=0 → passes as unknown (avoid mass exclusion)", @@ -55,40 +71,44 @@ func TestCountDevicesWithMinMemoryFromList(t *testing.T) { mk("/dev/nvidia0", 0), mk("/dev/nvidia1", 24576), }, - shared: false, minMemMb: 40000, want: 1, // unknown stays in, 24GiB excluded + sharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, minMemMb: 40000, want: 1, // unknown stays in, 24GiB excluded }, { - name: "MPS share: 2 physical cards, 4 slices each, only 1 card meets req", + name: "UNLIMITED share: 2 physical cards, only one card's slots meet req", devs: []*core.IsolatedDeviceDesc{ - // card 0: 6 GiB per slice (4 slices × same path) - mk("/dev/nvidia0", 6144), mk("/dev/nvidia0", 6144), - mk("/dev/nvidia0", 6144), mk("/dev/nvidia0", 6144), - // card 1: 20 GiB per slice - mk("/dev/nvidia1", 20480), mk("/dev/nvidia1", 20480), - mk("/dev/nvidia1", 20480), mk("/dev/nvidia1", 20480), + mkVirtual("/dev/nvidia0", 6144, 4, 0), + mkVirtual("/dev/nvidia1", 20480, 4, 0), }, - shared: true, minMemMb: 10000, want: 1, // only card 1 satisfies + sharingMode: computeapi.DEVICE_SHARING_MODE_UNLIMITED, minMemMb: 10000, want: 1, }, { - name: "MPS share: all slices pass through dedup → count by DevicePath", + name: "UNLIMITED share: all matching virtual slots are counted", devs: []*core.IsolatedDeviceDesc{ - mk("/dev/nvidia0", 24576), mk("/dev/nvidia0", 24576), - mk("/dev/nvidia1", 24576), + mkVirtual("/dev/nvidia0", 24576, 2, 0), + mkVirtual("/dev/nvidia1", 24576, 2, 1), }, - shared: true, minMemMb: 10000, want: 2, // 2 distinct paths + sharingMode: computeapi.DEVICE_SHARING_MODE_UNLIMITED, minMemMb: 10000, want: 2, }, { - name: "empty pool → 0", - devs: []*core.IsolatedDeviceDesc{}, - shared: false, - minMemMb: 1000, - want: 0, + name: "HAMI share: count cards with enough remaining memory", + devs: []*core.IsolatedDeviceDesc{ + mkHAMI("/dev/nvidia0", 24576, 8192), + mkHAMI("/dev/nvidia1", 24576, 20480), + }, + sharingMode: computeapi.DEVICE_SHARING_MODE_HAMI, minMemMb: 10000, want: 1, + }, + { + name: "empty pool → 0", + devs: []*core.IsolatedDeviceDesc{}, + sharingMode: computeapi.DEVICE_SHARING_MODE_EXCLUSIVE, + minMemMb: 1000, + want: 0, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - got := countDevicesWithMinMemoryFromList(c.devs, c.shared, c.minMemMb) + got := countDevicesWithMinMemoryFromList(c.devs, c.sharingMode, c.minMemMb) if got != c.want { t.Errorf("got %d, want %d", got, c.want) } diff --git a/pkg/scheduler/algorithm/predicates/network_predicate.go b/pkg/scheduler/algorithm/predicates/network_predicate.go index f9a4210e46..7c6a4a22f8 100644 --- a/pkg/scheduler/algorithm/predicates/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/network_predicate.go @@ -177,7 +177,7 @@ func checkSriovNic( } } getter := c.Getter() - devs := getter.UnusedIsolatedDevicesByModelAndWire(dev.Model, netWireId) + devs := getter.AvailableIsolatedDevicesByModelAndWire(dev.Model, netWireId) if len(devs) == 0 { return fmt.Errorf("Network wire no sriov nic available") } diff --git a/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go index 253ba04de9..e1b27f19d3 100644 --- a/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go +++ b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go @@ -52,7 +52,7 @@ func (p *GPUVramBinpackPriority) Map(u *core.Unit, c core.Candidater) (core.Host return h.GetResult() } - score := scoreVramBinpack(c.Getter().UnusedIsolatedDevices(), perDevMin) + score := scoreVramBinpack(c.Getter().AvailableIsolatedDevices(), perDevMin) h.SetScore(score) return h.GetResult() } diff --git a/pkg/scheduler/api/types.go b/pkg/scheduler/api/types.go index d8e7d2a15b..2558cf7d54 100644 --- a/pkg/scheduler/api/types.go +++ b/pkg/scheduler/api/types.go @@ -55,32 +55,8 @@ var ( AggregateStrategyPrefer, AggregateStrategyAvoid, ) - - ValidGpuTypes = sets.NewString( - GPU_HPC_TYPE, - GPU_VGA_TYPE, - ) - - ValidPassthroughTypes = sets.NewString( - DIRECT_PCI_TYPE, - USB_TYPE, - NIC_TYPE, - ).Union(ValidGpuTypes) - - IsolatedVendorIDMap = map[string]string{ - NVIDIA: NVIDIA_VENDOR_ID, - AMD: AMD_VENDOR_ID, - } - - IsolatedIDVendorMap = map[string]string{} ) -func init() { - for k, v := range IsolatedVendorIDMap { - IsolatedIDVendorMap[v] = k - } -} - func SchedtagStrategyCheck(strategy string) (err error) { if !AggregateStrategySets.Has(strategy) { err = fmt.Errorf("Strategy %q must in set %v", strategy, AggregateStrategySets.List()) diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index 749f2759ab..18bdebc6cb 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -296,38 +296,34 @@ func (b baseHostGetter) GetPendingUsage() *schedmodels.SPendingUsage { return b.h.GetPendingUsage() } -func (b baseHostGetter) UnusedIsolatedDevices() []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevices() +func (b baseHostGetter) AvailableIsolatedDevices() []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevices() } -func (b baseHostGetter) UnusedIsolatedDevicesByType(devType string) []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevicesByType(devType) +func (b baseHostGetter) AvailableIsolatedDevicesByTypeSharingMode(devType string, sharingMode string) []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode) } -func (b baseHostGetter) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevicesByVendorModel(vendorModel) +func (b baseHostGetter) AvailableIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevicesByVendorModel(vendorModel) } -func (b baseHostGetter) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevicesByDevicePath(devPath) +func (b baseHostGetter) AvailableIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevicesByDevicePath(devPath) } -func (b baseHostGetter) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevicesByModel(model) +func (b baseHostGetter) AvailableIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevicesByModel(model) } -func (b baseHostGetter) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { - return b.h.UnusedIsolatedDevicesByModelAndWire(model, wire) +func (b baseHostGetter) AvailableIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { + return b.h.AvailableIsolatedDevicesByModelAndWire(model, wire) } func (b baseHostGetter) GetIsolatedDevice(devID string) *core.IsolatedDeviceDesc { return b.h.GetIsolatedDevice(devID) } -func (b baseHostGetter) UnusedGpuDevices() []*core.IsolatedDeviceDesc { - return b.h.UnusedGpuDevices() -} - func (b baseHostGetter) GetIsolatedDevices() []*core.IsolatedDeviceDesc { return b.h.GetIsolatedDevices() } @@ -485,30 +481,34 @@ func (b BaseHostDesc) GetResourceType() string { return b.ResourceType } -func (h *BaseHostDesc) UnusedIsolatedDevices() []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevices() []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) for _, dev := range h.IsolatedDevices { - if len(dev.GuestID) == 0 { + if !dev.IsUsedUp() { ret = append(ret, dev) } } return ret } -func (h *BaseHostDesc) UnusedIsolatedDevicesByType(devType string) []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { - if dev.DevType == devType { - ret = append(ret, dev) + for _, dev := range h.AvailableIsolatedDevices() { + if devType != "" && dev.DevType != devType { + continue } + if sharingMode != "" && dev.SharingMode != sharingMode { + continue + } + ret = append(ret, dev) } return ret } -func (h *BaseHostDesc) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) vm := core.NewVendorModelByStr(vendorModel) - for _, dev := range h.UnusedIsolatedDevices() { + for _, dev := range h.AvailableIsolatedDevices() { if dev.GetVendorModel().IsMatch(vm) { ret = append(ret, dev) } @@ -516,9 +516,9 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByVendorModel(vendorModel string) [] return ret } -func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { + for _, dev := range h.AvailableIsolatedDevices() { if strings.Contains(dev.Model, model) { ret = append(ret, dev) } @@ -526,9 +526,9 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.Isolat return ret } -func (h *BaseHostDesc) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { + for _, dev := range h.AvailableIsolatedDevices() { if devPath == dev.DevicePath { ret = append(ret, dev) } @@ -536,9 +536,9 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByDevicePath(devPath string) []*core return ret } -func (h *BaseHostDesc) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { +func (h *BaseHostDesc) AvailableIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { + for _, dev := range h.AvailableIsolatedDevices() { log.Errorf("dev wire is %s, dev model is %s, request model is %s, request wire is %s", dev.Model, dev.WireId, model, wire) if strings.Contains(dev.Model, model) && dev.WireId == wire { ret = append(ret, dev) @@ -560,16 +560,6 @@ func (h *BaseHostDesc) GetIsolatedDevices() []*core.IsolatedDeviceDesc { return h.IsolatedDevices } -func (h *BaseHostDesc) UnusedGpuDevices() []*core.IsolatedDeviceDesc { - ret := make([]*core.IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { - if strings.HasPrefix(dev.DevType, "GPU") { - ret = append(ret, dev) - } - } - return ret -} - func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.SHost) error { allDevs := b.getIsolatedDevices(host.Id) if len(allDevs) == 0 { @@ -579,16 +569,19 @@ func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.S devs := make([]*core.IsolatedDeviceDesc, len(allDevs)) for index, devModel := range allDevs { dev := &core.IsolatedDeviceDesc{ - ID: devModel.Id, - GuestID: devModel.GuestId, - HostID: devModel.HostId, - DevType: devModel.DevType, - Model: devModel.Model, - Addr: devModel.Addr, - VendorDeviceID: devModel.VendorDeviceId, - WireId: devModel.WireId, - DevicePath: devModel.DevicePath, - MemorySize: devModel.MemorySize, + ID: devModel.Id, + HostID: devModel.HostId, + DevType: devModel.DevType, + SharingMode: devModel.SharingMode, + Model: devModel.Model, + Addr: devModel.Addr, + VendorDeviceID: devModel.VendorDeviceId, + WireId: devModel.WireId, + DevicePath: devModel.DevicePath, + MemorySize: devModel.MemorySize, + MemorySizeAllocated: devModel.MemoryAllocated, + VirtualNum: devModel.VirtualNum, + VirtualNumAllocated: devModel.GuestCount, } devs[index] = dev } diff --git a/pkg/scheduler/cache/candidate/builder.go b/pkg/scheduler/cache/candidate/builder.go index 0536c5aabc..670ff6fb5c 100644 --- a/pkg/scheduler/cache/candidate/builder.go +++ b/pkg/scheduler/cache/candidate/builder.go @@ -43,7 +43,8 @@ type baseBuilder struct { hosts []computemodels.SHost hostDict map[string]*computemodels.SHost - isolatedDevicesDict map[string][]interface{} + isolatedDevicesDict map[string][]interface{} + isolatedDeviceGuestsDict map[string][]string hostCloudproviers map[string]*computemodels.SCloudprovider hostCloudaccounts map[string]*computemodels.SCloudaccount @@ -181,23 +182,23 @@ func (b *baseBuilder) setHosts(ids []string) error { return nil } -func (b *baseBuilder) getIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { +func (b *baseBuilder) getIsolatedDevices(hostID string) (devs []computemodels.IsolatedDeviceAllocateStat) { devObjs, ok := b.isolatedDevicesDict[hostID] - devs = make([]computemodels.SIsolatedDevice, 0) + devs = make([]computemodels.IsolatedDeviceAllocateStat, 0) if !ok { return } for _, obj := range devObjs { - dev := obj.(computemodels.SIsolatedDevice) + dev := obj.(computemodels.IsolatedDeviceAllocateStat) devs = append(devs, dev) } return } func (b *baseBuilder) setIsolatedDevs(ids []string, errMessageChannel chan error) { - devs := computemodels.IsolatedDeviceManager.FindByHosts(ids) + devs := computemodels.IsolatedDeviceManager.GetHostsIsolatedDeviceStats(ids) dict, err := utils.GroupBy(devs, func(obj interface{}) (string, error) { - dev, ok := obj.(computemodels.SIsolatedDevice) + dev, ok := obj.(computemodels.IsolatedDeviceAllocateStat) if !ok { return "", utils.ConvertError(obj, "computemodels.SIsolatedDevice") } @@ -208,6 +209,8 @@ func (b *baseBuilder) setIsolatedDevs(ids []string, errMessageChannel chan error return } b.isolatedDevicesDict = dict + devGuests := computemodels.IsolatedDeviceManager.GetHostsGuestIsolatedDevices(ids) + b.isolatedDeviceGuestsDict = devGuests } func (b *baseBuilder) setCloudproviderAccounts(hosts []computemodels.SHost, errCh chan error) { diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 2f59e9a798..f200753d45 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -814,7 +814,8 @@ func NewGuestReservedResourceByBuilder(b *HostBuilder, host *computemodels.SHost if len(isoDevs) == 0 { return } - reservedResource := host.GetDevsReservedResource(isoDevs) + + reservedResource := host.GetDevsReservedResourceByDevStats(isoDevs) if reservedResource != nil { ret.CPUCount = int64(*reservedResource.ReservedCpu) ret.MemorySize = int64(*reservedResource.ReservedMemory) @@ -1744,25 +1745,25 @@ func (b *HostBuilder) fillMetadata(desc *HostDesc, host *computemodels.SHost) er return nil } -func (b *HostBuilder) getUsedIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { - devs = make([]computemodels.SIsolatedDevice, 0) +func (b *HostBuilder) getUsedIsolatedDevicesGuests(hostID string) []string { + guests := make([]string, 0) for _, dev := range b.getIsolatedDevices(hostID) { - if len(dev.GuestId) != 0 { - devs = append(devs, dev) + if gss, ok := b.isolatedDeviceGuestsDict[dev.Id]; ok { + guests = append(guests, gss...) } } - return + return guests } func (b *HostBuilder) getIsolatedDeviceGuests(hostID string) (guests []computemodels.SGuest) { guests = make([]computemodels.SGuest, 0) - usedDevs := b.getUsedIsolatedDevices(hostID) - if len(usedDevs) == 0 { + usedGuests := b.getUsedIsolatedDevicesGuests(hostID) + if len(usedGuests) == 0 { return } ids := sets.NewString() - for _, dev := range usedDevs { - g, ok := b.guestDict[dev.GuestId] + for _, guestId := range usedGuests { + g, ok := b.guestDict[guestId] if !ok { continue } @@ -1775,15 +1776,15 @@ func (b *HostBuilder) getIsolatedDeviceGuests(hostID string) (guests []computemo return } -func (b *HostBuilder) getUnusedIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { - devs = make([]computemodels.SIsolatedDevice, 0) - for _, dev := range b.getIsolatedDevices(hostID) { - if len(dev.GuestId) == 0 { - devs = append(devs, dev) - } - } - return -} +//func (b *HostBuilder) getUnusedIsolatedDevices(hostID string) (devs []computemodels.IsolatedDeviceAllocateStat) { +// devs = make([]computemodels.IsolatedDeviceAllocateStat, 0) +// for _, dev := range b.getIsolatedDevices(hostID) { +// if len(dev.GuestId) == 0 { +// devs = append(devs, dev) +// } +// } +// return +//} func (b *HostBuilder) fillCPUIOLoads(desc *HostDesc, host *computemodels.SHost) error { desc.CPULoad = b.loadByName(host.Id, "cpu_load") diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index b552a93b99..9f36da133b 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -121,14 +121,13 @@ type CandidatePropertyGetter interface { GetPendingUsage() *schedmodels.SPendingUsage // isloatedDevices - UnusedIsolatedDevices() []*IsolatedDeviceDesc - UnusedIsolatedDevicesByType(devType string) []*IsolatedDeviceDesc - UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc - UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc - UnusedIsolatedDevicesByModelAndWire(model, wire string) []*IsolatedDeviceDesc - UnusedIsolatedDevicesByDevicePath(devPath string) []*IsolatedDeviceDesc + AvailableIsolatedDevices() []*IsolatedDeviceDesc + AvailableIsolatedDevicesByTypeSharingMode(devType string, sharingMode string) []*IsolatedDeviceDesc + AvailableIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc + AvailableIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc + AvailableIsolatedDevicesByModelAndWire(model, wire string) []*IsolatedDeviceDesc + AvailableIsolatedDevicesByDevicePath(devPath string) []*IsolatedDeviceDesc GetIsolatedDevice(devID string) *IsolatedDeviceDesc - UnusedGpuDevices() []*IsolatedDeviceDesc GetIsolatedDevices() []*IsolatedDeviceDesc db.IResource @@ -273,6 +272,7 @@ type IsolatedDeviceDesc struct { GuestID string HostID string DevType string + SharingMode string Model string Addr string VendorDeviceID string @@ -280,7 +280,26 @@ type IsolatedDeviceDesc struct { DevicePath string // MemorySize is the on-device memory in MiB (NVIDIA GPU VRAM via // `nvidia-smi memory.total`). 0 means unknown / not yet reported by host. - MemorySize int + MemorySize int + MemorySizeAllocated int + VirtualNum int + VirtualNumAllocated int +} + +func (i *IsolatedDeviceDesc) IsUsedUp() bool { + if i.SharingMode == computeapi.DEVICE_SHARING_MODE_HAMI { + return i.MemorySizeAllocated >= i.MemorySize + } else { + return i.VirtualNumAllocated >= i.VirtualNum + } +} + +func (i *IsolatedDeviceDesc) AvailableNum() int { + return i.VirtualNum - i.VirtualNumAllocated +} + +func (i *IsolatedDeviceDesc) AvailableMemorySize() int { + return i.MemorySize - i.MemorySizeAllocated } func (i *IsolatedDeviceDesc) VendorID() string { diff --git a/pkg/scheduler/models/pending_usage.go b/pkg/scheduler/models/pending_usage.go index 4a575aafe6..5383fcdd7e 100644 --- a/pkg/scheduler/models/pending_usage.go +++ b/pkg/scheduler/models/pending_usage.go @@ -17,6 +17,7 @@ package models import ( "context" "fmt" + "path" "sync" "time" @@ -24,6 +25,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/compute" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -512,7 +514,7 @@ type SPendingUsage struct { // nodeId: memSizeMB NumaMemPin map[int]int - IsolatedDevice int + IsolatedDevice *SResourcePendingUsage DiskUsage *SResourcePendingUsage NetUsage *SResourcePendingUsage // Lock is not need here @@ -524,6 +526,7 @@ func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo, candidate *sc HostId: hostId, DiskUsage: NewResourcePendingUsage(nil), NetUsage: NewResourcePendingUsage(nil), + IsolatedDevice: NewResourcePendingUsage(nil), PendingGuestIds: make(map[string]struct{}), } @@ -537,7 +540,6 @@ func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo, candidate *sc } u.Cpu = req.Ncpu u.Memory = req.Memory - u.IsolatedDevice = len(req.IsolatedDevices) if candidate != nil && len(candidate.CpuNumaPin) > 0 { for _, cpuNumaPin := range candidate.CpuNumaPin { @@ -559,6 +561,20 @@ func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo, candidate *sc } } + for _, dev := range req.IsolatedDevices { + devType := dev.DevType + sharingMode := dev.SharingMode + pendingKey := path.Join(devType, sharingMode) + if sharingMode == compute.DEVICE_SHARING_MODE_HAMI { + oSize := u.IsolatedDevice.Get(pendingKey) + size := dev.MemoryRequest + u.IsolatedDevice.Set(pendingKey, oSize+size) + } else { + oCnt := u.IsolatedDevice.Get(pendingKey) + u.IsolatedDevice.Set(pendingKey, oCnt+1) + } + } + for _, disk := range req.Disks { backend := disk.Backend size := disk.SizeMb @@ -608,7 +624,7 @@ func (self *SPendingUsage) ToMap() map[string]interface{} { return map[string]interface{}{ "cpu": self.Cpu, "memory": self.Memory, - "isolated_device": self.IsolatedDevice, + "isolated_device": self.IsolatedDevice.ToMap(), "disk": self.DiskUsage.ToMap(), "net": self.NetUsage.ToMap(), "instance_groups": self.InstanceGroupUsage, @@ -644,7 +660,7 @@ func (self *SPendingUsage) Add(sUsage *SPendingUsage, addGuestId string) { self.NumaMemPin[k] = v1 } } - self.IsolatedDevice = self.IsolatedDevice + sUsage.IsolatedDevice + self.IsolatedDevice.Add(sUsage.IsolatedDevice) self.DiskUsage.Add(sUsage.DiskUsage) self.NetUsage.Add(sUsage.NetUsage) for id, cg := range sUsage.InstanceGroupUsage { @@ -676,7 +692,7 @@ func (self *SPendingUsage) Sub(sUsage *SPendingUsage) { } } - self.IsolatedDevice = quotas.NonNegative(self.IsolatedDevice - sUsage.IsolatedDevice) + self.IsolatedDevice.Sub(sUsage.IsolatedDevice) self.DiskUsage.Sub(sUsage.DiskUsage) self.NetUsage.Sub(sUsage.NetUsage) for id, cg := range sUsage.InstanceGroupUsage { @@ -698,7 +714,7 @@ func (self *SPendingUsage) IsEmpty() bool { if self.Memory > 0 { return false } - if self.IsolatedDevice > 0 { + if !self.IsolatedDevice.IsEmpty() { return false } if !self.DiskUsage.IsEmpty() { diff --git a/pkg/scheduler/test/mock/core.go b/pkg/scheduler/test/mock/core.go index f15599f450..7e8e6af8b9 100644 --- a/pkg/scheduler/test/mock/core.go +++ b/pkg/scheduler/test/mock/core.go @@ -697,96 +697,82 @@ func (mr *MockCandidatePropertyGetterMockRecorder) TotalMemorySize(arg0 interfac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TotalMemorySize", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).TotalMemorySize), arg0) } -// UnusedGpuDevices mocks base method -func (m *MockCandidatePropertyGetter) UnusedGpuDevices() []*core.IsolatedDeviceDesc { +// AvailableIsolatedDevices mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevices() []*core.IsolatedDeviceDesc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedGpuDevices") + ret := m.ctrl.Call(m, "AvailableIsolatedDevices") ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) return ret0 } -// UnusedGpuDevices indicates an expected call of UnusedGpuDevices -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedGpuDevices() *gomock.Call { +// AvailableIsolatedDevices indicates an expected call of AvailableIsolatedDevices +func (mr *MockCandidatePropertyGetterMockRecorder) AvailableIsolatedDevices() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedGpuDevices", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedGpuDevices)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolatedDevices", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).AvailableIsolatedDevices)) } -// UnusedIsolatedDevices mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevices() []*core.IsolatedDeviceDesc { +// AvailableIsolatedDevicesByModel mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevicesByModel(arg0 string) []*core.IsolatedDeviceDesc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevices") + ret := m.ctrl.Call(m, "AvailableIsolatedDevicesByModel", arg0) ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) return ret0 } -// UnusedIsolatedDevices indicates an expected call of UnusedIsolatedDevices -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevices() *gomock.Call { +// AvailableIsolatedDevicesByDevicePath mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevicesByDevicePath(arg0 string) []*core.IsolatedDeviceDesc { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AvailableIsolatedDevicesByDevicePath", arg0) + ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) + return ret0 +} + +// AvailableIsolatedDevicesByModel indicates an expected call of AvailableIsolatedDevicesByModel +func (mr *MockCandidatePropertyGetterMockRecorder) AvailableIsolatedDevicesByModel(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevices", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevices)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolatedDevicesByModel", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).AvailableIsolatedDevicesByModel), arg0) } -// UnusedIsolatedDevicesByModel mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByModel(arg0 string) []*core.IsolatedDeviceDesc { +// AvailableIsolatedDevicesByModelAndWire mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevicesByModelAndWire(arg0, arg1 string) []*core.IsolatedDeviceDesc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByModel", arg0) + ret := m.ctrl.Call(m, "AvailableIsolatedDevicesByModelAndWire", arg0, arg1) ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) return ret0 } -// UnusedIsolatedDevicesByDevicePath mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByDevicePath(arg0 string) []*core.IsolatedDeviceDesc { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByDevicePath", arg0) - ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) - return ret0 -} - -// UnusedIsolatedDevicesByModel indicates an expected call of UnusedIsolatedDevicesByModel -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModel(arg0 interface{}) *gomock.Call { +// AvailableIsolatedDevicesByModelAndWire indicates an expected call of AvailableIsolatedDevicesByModelAndWire +func (mr *MockCandidatePropertyGetterMockRecorder) AvailableIsolatedDevicesByModelAndWire(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByModel", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByModel), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolatedDevicesByModelAndWire", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).AvailableIsolatedDevicesByModelAndWire), arg0, arg1) } -// UnusedIsolatedDevicesByModelAndWire mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByModelAndWire(arg0, arg1 string) []*core.IsolatedDeviceDesc { +// AvailableIsolatedDevicesByTypeSharingMode mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevicesByTypeSharingMode(arg0 string, arg1 string) []*core.IsolatedDeviceDesc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByModelAndWire", arg0, arg1) + ret := m.ctrl.Call(m, "AvailableIsolatedDevicesByType", arg0, arg1) ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) return ret0 } -// UnusedIsolatedDevicesByModelAndWire indicates an expected call of UnusedIsolatedDevicesByModelAndWire -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModelAndWire(arg0, arg1 interface{}) *gomock.Call { +// AvailableIsolatedDevicesByTypeSharingMode indicates an expected call of AvailableIsolatedDevicesByTypeSharingMode +func (mr *MockCandidatePropertyGetterMockRecorder) AvailableIsolatedDevicesByTypeSharingMode(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByModelAndWire", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByModel), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolatedDevicesByType", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).AvailableIsolatedDevicesByTypeSharingMode), arg0, arg1) } -// UnusedIsolatedDevicesByType mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByType(arg0 string) []*core.IsolatedDeviceDesc { +// AvailableIsolatedDevicesByVendorModel mocks base method +func (m *MockCandidatePropertyGetter) AvailableIsolatedDevicesByVendorModel(arg0 string) []*core.IsolatedDeviceDesc { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByType", arg0) + ret := m.ctrl.Call(m, "AvailableIsolatedDevicesByVendorModel", arg0) ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) return ret0 } -// UnusedIsolatedDevicesByType indicates an expected call of UnusedIsolatedDevicesByType -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByType(arg0 interface{}) *gomock.Call { +// AvailableIsolatedDevicesByVendorModel indicates an expected call of AvailableIsolatedDevicesByVendorModel +func (mr *MockCandidatePropertyGetterMockRecorder) AvailableIsolatedDevicesByVendorModel(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByType", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByType), arg0) -} - -// UnusedIsolatedDevicesByVendorModel mocks base method -func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByVendorModel(arg0 string) []*core.IsolatedDeviceDesc { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByVendorModel", arg0) - ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) - return ret0 -} - -// UnusedIsolatedDevicesByVendorModel indicates an expected call of UnusedIsolatedDevicesByVendorModel -func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByVendorModel(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnusedIsolatedDevicesByVendorModel", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).UnusedIsolatedDevicesByVendorModel), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailableIsolatedDevicesByVendorModel", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).AvailableIsolatedDevicesByVendorModel), arg0) } // Zone mocks base method