Files
cloudpods/pkg/hostman/guestman/pci.go

1144 lines
35 KiB
Go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guestman
import (
"fmt"
"strconv"
"strings"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
"yunion.io/x/onecloud/pkg/hostman/guestman/qemu"
"yunion.io/x/onecloud/pkg/hostman/monitor"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/util/fileutils2"
)
func (s *SKVMGuestInstance) addPCIController(controllerType, bus desc.PCI_CONTROLLER_TYPE) *desc.PCIController {
cont := desc.NewPCIController(controllerType)
cont.PCIDevice = desc.NewPCIDevice(bus, "", "")
if s.Desc.PCIControllers == nil {
s.Desc.PCIControllers = make([]*desc.PCIController, 0)
}
s.Desc.PCIControllers = append(s.Desc.PCIControllers, cont)
return cont
}
func (s *SKVMGuestInstance) pciControllerFind(cont desc.PCI_CONTROLLER_TYPE) *desc.PCIController {
for i := 0; i < len(s.Desc.PCIControllers); i++ {
if s.Desc.PCIControllers[i].CType == cont {
return s.Desc.PCIControllers[i]
}
}
return nil
}
func (s *SKVMGuestInstance) initGuestDesc() error {
err := s.initCpuDesc(0)
if err != nil {
return err
}
err = s.initMemDesc(s.Desc.Mem)
if err != nil {
return errors.Wrap(err, "initMemDesc")
}
s.initMachineDesc()
pciRoot, pciBridge := s.initGuestPciControllers(s.manager.host.IsKvmSupport())
err = s.initGuestPciAddresses()
if err != nil {
return errors.Wrap(err, "init guest pci addresses")
}
err = s.initGuestDevicesDesc(pciRoot, pciBridge)
if err != nil {
return err
}
if err := s.initMachineDefaultAddresses(); err != nil {
return errors.Wrap(err, "init machine default devices")
}
return s.ensurePciAddresses()
}
func (s *SKVMGuestInstance) initGuestDevicesDesc(pciRoot, pciBridge *desc.PCIController) error {
// vdi device for spice
s.Desc.VdiDevice = new(desc.SGuestVdi)
if s.IsVdiSpice() {
s.initSpiceDevices(pciRoot)
}
s.initVirtioSerial(pciRoot)
s.initGuestVga(pciRoot)
s.initCdromDesc()
s.initFloppyDesc()
s.initGuestDisks(pciRoot, pciBridge, false)
if err := s.initGuestNetworks(pciRoot, pciBridge); err != nil {
return errors.Wrap(err, "init guest networks")
}
s.initIsolatedDevices(pciRoot, pciBridge)
s.initUsbController(pciRoot)
s.initRandomDevice(pciRoot, options.HostOptions.EnableVirtioRngDevice)
s.initQgaDesc()
s.initPvpanicDesc()
s.initIsaSerialDesc()
s.initTpmDesc()
return nil
}
func (s *SKVMGuestInstance) loadGuestPciAddresses() error {
err := s.initGuestPciAddresses()
if err != nil {
return errors.Wrap(err, "init guest pci addresses")
}
if err := s.initMachineDefaultAddresses(); err != nil {
return errors.Wrap(err, "init machine default devices")
}
err = s.ensurePciAddresses()
if err != nil {
return errors.Wrap(err, "load desc ensure pci address")
}
if err = SaveLiveDesc(s, s.Desc); err != nil {
return errors.Wrap(err, "loadGuestPciAddresses save desc")
}
return nil
}
func (s *SKVMGuestInstance) initMachineDefaultAddresses() error {
switch s.Desc.Machine {
case "pc":
I440FXSlot1 := &desc.PCIDevice{
PCIAddr: &desc.PCIAddr{Bus: 0, Slot: 1},
Controller: desc.CONTROLLER_TYPE_PCI_ROOT,
}
// ISA bridge
multiFunc := true
if err := s.ensureDevicePciAddress(I440FXSlot1, 0, &multiFunc); err != nil {
return err
}
// primary IDE controller
if err := s.ensureDevicePciAddress(I440FXSlot1, 1, nil); err != nil {
return err
}
// PIIX3 USB controller
if err := s.ensureDevicePciAddress(I440FXSlot1, 2, nil); err != nil {
return err
}
// ACPI (power management) and SMBus controller
if err := s.ensureDevicePciAddress(I440FXSlot1, 3, nil); err != nil {
return err
}
case "q35":
Q35Slot31 := &desc.PCIDevice{
PCIAddr: &desc.PCIAddr{Bus: 0, Slot: 31},
Controller: desc.CONTROLLER_TYPE_PCIE_ROOT,
}
// ISA bridge
multiFunc := true
if err := s.ensureDevicePciAddress(Q35Slot31, 0, &multiFunc); err != nil {
return err
}
// primary SATA controller
if err := s.ensureDevicePciAddress(Q35Slot31, 2, nil); err != nil {
return err
}
// SMBus
if err := s.ensureDevicePciAddress(Q35Slot31, 3, nil); err != nil {
return err
}
// usb controllers
Q35Slot29 := &desc.PCIDevice{
PCIAddr: &desc.PCIAddr{Bus: 0, Slot: 29},
Controller: desc.CONTROLLER_TYPE_PCIE_ROOT,
}
multiFunc = true
if err := s.ensureDevicePciAddress(Q35Slot29, 0, &multiFunc); err != nil {
return err
}
if err := s.ensureDevicePciAddress(Q35Slot29, 1, nil); err != nil {
return err
}
if err := s.ensureDevicePciAddress(Q35Slot29, 2, nil); err != nil {
return err
}
if err := s.ensureDevicePciAddress(Q35Slot29, 7, nil); err != nil {
return err
}
case "virt":
// do nothing
}
return nil
}
func (s *SKVMGuestInstance) isMachineDefaultAddress(pciAddr *desc.PCIAddr) bool {
switch s.Desc.Machine {
case "pc":
if pciAddr.Bus == 0 && pciAddr.Slot == 1 && pciAddr.Function < 4 {
return true
}
case "q35":
if pciAddr.Bus == 0 && pciAddr.Slot == 29 {
switch pciAddr.Function {
case 0, 1, 2, 7:
return true
}
} else if pciAddr.Bus == 0 && pciAddr.Slot == 31 {
switch pciAddr.Function {
case 0, 2, 3:
return true
}
}
}
return false
}
func (s *SKVMGuestInstance) initGuestPciControllers(pciExtend bool) (*desc.PCIController, *desc.PCIController) {
var pciRoot, pciBridge *desc.PCIController
if len(s.Desc.PCIControllers) > 0 {
pciRoot = s.Desc.PCIControllers[0]
if len(s.Desc.PCIControllers) > 1 {
pciBridge = s.Desc.PCIControllers[1]
}
return pciRoot, pciBridge
}
if s.isPcie() {
pciRoot = s.addPCIController(desc.CONTROLLER_TYPE_PCIE_ROOT, "")
if pciExtend {
s.addPCIController(desc.CONTROLLER_TYPE_PCIE_TO_PCI_BRIDGE, desc.CONTROLLER_TYPE_PCIE_ROOT)
pciBridge = s.addPCIController(desc.CONTROLLER_TYPE_PCI_BRIDGE, desc.CONTROLLER_TYPE_PCIE_TO_PCI_BRIDGE)
for i := 0; i < s.vfioDevCount()+options.HostOptions.PcieRootPortCount; i++ {
s.addPCIController(desc.CONTROLLER_TYPE_PCIE_ROOT_PORT, desc.CONTROLLER_TYPE_PCIE_ROOT)
}
}
} else {
pciRoot = s.addPCIController(desc.CONTROLLER_TYPE_PCI_ROOT, "")
}
return pciRoot, pciBridge
}
func (s *SKVMGuestInstance) initGuestPciAddresses() error {
s.pciAddrs = &desc.SGuestPCIAddresses{
Buses: make([]*desc.SGuestPCIAddressBus, len(s.Desc.PCIControllers)),
}
var err error
for i := 0; i < len(s.pciAddrs.Buses); i++ {
s.pciAddrs.Buses[i], err = desc.NewGuestPCIAddressBus(s.Desc.PCIControllers[i].CType)
if err != nil {
return err
}
}
return nil
}
func (s *SKVMGuestInstance) cleanGuestPciAddressed() {
s.pciAddrs = nil
}
func (s *SKVMGuestInstance) initRandomDevice(pciRoot *desc.PCIController, enableVirtioRngDevice bool) {
if !enableVirtioRngDevice {
return
}
var randev string
if fileutils2.Exists("/dev/urandom") {
randev = "/dev/urandom"
} else if fileutils2.Exists("/dev/random") {
randev = "/dev/random"
} else {
return
}
s.Desc.Rng = &desc.SGuestRng{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "virtio-rng-pci", "random0"),
RngRandom: desc.NewObject("rng-random", "rng0"),
}
s.Desc.Rng.Options = map[string]string{
"max-bytes": "1024",
"period": "1000",
}
s.Desc.Rng.RngRandom.Options = map[string]string{
"filename": randev,
}
}
func (s *SKVMGuestInstance) initUsbController(pciRoot *desc.PCIController) {
contType := s.getUsbControllerType()
s.Desc.Usb = &desc.UsbController{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, contType, "usb"),
}
}
func (s *SKVMGuestInstance) initGuestNetworks(pciRoot, pciBridge *desc.PCIController) error {
cont := pciRoot
if pciBridge != nil {
cont = pciBridge
}
if s.GetOsName() == OS_NAME_ANDROID {
s.Desc.Nics = s.Desc.Nics[:1]
}
for i := 0; i < len(s.Desc.Nics); i++ {
if err := s.generateNicScripts(s.Desc.Nics[i]); err != nil {
return errors.Wrapf(err, "generateNicScripts for nic: %v", s.Desc.Nics[i])
}
s.Desc.Nics[i].UpscriptPath = s.getNicUpScriptPath(s.Desc.Nics[i])
s.Desc.Nics[i].DownscriptPath = s.getNicDownScriptPath(s.Desc.Nics[i])
if s.Desc.Nics[i].Driver != "vfio-pci" {
switch s.GetOsName() {
case OS_NAME_MACOS:
vectors := 0
s.Desc.Nics[i].Vectors = &vectors
s.Desc.Nics[i].Driver = "e1000"
case OS_NAME_VMWARE:
s.Desc.Nics[i].Driver = "vmxnet3"
}
if s.Desc.Nics[i].NumQueues > 1 {
vectors := s.Desc.Nics[i].NumQueues*2 + 1
s.Desc.Nics[i].Vectors = &vectors
}
id := fmt.Sprintf("netdev-%s", s.Desc.Nics[i].Ifname)
switch s.Desc.Nics[i].Driver {
case "virtio":
s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "virtio-net-pci", id)
case "e1000":
s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "e1000-82545em", id)
case "vmxnet3":
s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "vmxnet3", id)
case "rtl8139":
s.Desc.Nics[i].Pci = desc.NewPCIDevice(cont.CType, "rtl8139", id)
}
}
}
return nil
}
func (s *SKVMGuestInstance) initIsolatedDevices(pciRoot, pciBridge *desc.PCIController) {
cType := s.getVfioDeviceHotPlugPciControllerType()
if cType == nil {
cType = &pciRoot.CType
if pciBridge != nil {
cType = &pciBridge.CType
}
}
manager := s.manager.GetHost().GetIsolatedDeviceManager()
for i := 0; i < len(s.Desc.IsolatedDevices); i++ {
dev := manager.GetDeviceByAddr(s.Desc.IsolatedDevices[i].Addr)
if s.Desc.IsolatedDevices[i].DevType == compute.USB_TYPE {
s.Desc.IsolatedDevices[i].Usb = desc.NewUsbDevice("usb-host", dev.GetQemuId())
s.Desc.IsolatedDevices[i].Usb.Options = dev.GetPassthroughOptions()
} else {
id := dev.GetQemuId()
s.Desc.IsolatedDevices[i].VfioDevs = make([]*desc.VFIODevice, 0)
vfioDev := desc.NewVfioDevice(
*cType, "vfio-pci", id, dev.GetAddr(), s.Desc.IsolatedDevices[i].GpuType == compute.GPU_VGA,
)
s.Desc.IsolatedDevices[i].VfioDevs = append(s.Desc.IsolatedDevices[i].VfioDevs, vfioDev)
groupDevAddrs := dev.GetIOMMUGroupRestAddrs()
for j := 0; j < len(groupDevAddrs); j++ {
gid := fmt.Sprintf("%s-%d", id, j+1)
vfioDev = desc.NewVfioDevice(*cType, "vfio-pci", gid, groupDevAddrs[j], false)
s.Desc.IsolatedDevices[i].VfioDevs = append(
s.Desc.IsolatedDevices[i].VfioDevs, vfioDev,
)
}
}
}
}
func (s *SKVMGuestInstance) initCdromDesc() {
if s.Desc.Cdroms == nil {
s.Desc.Cdroms = make([]*desc.SGuestCdrom, options.HostOptions.CdromCount)
for i := range s.Desc.Cdroms {
s.Desc.Cdroms[i] = new(desc.SGuestCdrom)
s.Desc.Cdroms[i].Ordinal = int64(i)
}
}
for i := range s.Desc.Cdroms {
s.archMan.GenerateCdromDesc(s.GetOsName(), s.Desc.Cdroms[i])
}
}
func (s *SKVMGuestInstance) initFloppyDesc() {
if s.Desc.Machine != "pc" || s.GetOsName() != OS_NAME_WINDOWS {
return
}
if s.Desc.Floppys == nil {
s.Desc.Floppys = make([]*desc.SGuestFloppy, options.HostOptions.FloppyCount)
for i := range s.Desc.Floppys {
s.Desc.Floppys[i] = new(desc.SGuestFloppy)
s.Desc.Floppys[i].Ordinal = int64(i)
}
}
for i := range s.Desc.Floppys {
s.archMan.GenerateFloppyDesc(s.GetOsName(), s.Desc.Floppys[i])
}
}
func (s *SKVMGuestInstance) initGuestDisks(pciRoot, pciBridge *desc.PCIController, loadGuest bool) {
if !loadGuest {
hasVirtioScsi, hasPvScsi, hasSataDisk := s.fixDiskDriver()
if hasVirtioScsi && s.Desc.VirtioScsi == nil {
s.Desc.VirtioScsi = &desc.SGuestVirtioScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "virtio-scsi-pci", "scsi"),
}
} else if hasPvScsi && s.Desc.PvScsi == nil {
s.Desc.PvScsi = &desc.SGuestPvScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "pvscsi", "scsi"),
}
} else if hasSataDisk {
s.Desc.SataController = &desc.SGuestAhciDevice{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "ahci", "ahci0"),
}
}
}
cont := pciRoot
if pciBridge != nil {
cont = pciBridge
}
for i := 0; i < len(s.Desc.Disks); i++ {
devType := qemu.GetDiskDeviceModel(s.Desc.Disks[i].Driver)
id := fmt.Sprintf("drive_%d", s.Desc.Disks[i].Index)
if s.Desc.Disks[i].Pci != nil || s.Desc.Disks[i].Scsi != nil {
log.Infof("guest %s disk %v has been init", s.Desc.Uuid, s.Desc.Disks[i].Index)
continue
}
switch s.Desc.Disks[i].Driver {
case DISK_DRIVER_VIRTIO:
if s.Desc.Disks[i].Pci == nil {
s.Desc.Disks[i].Pci = desc.NewPCIDevice(cont.CType, devType, id)
}
case DISK_DRIVER_SCSI:
if s.Desc.VirtioScsi == nil {
s.Desc.VirtioScsi = &desc.SGuestVirtioScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "virtio-scsi-pci", "scsi"),
}
}
s.Desc.Disks[i].Scsi = desc.NewScsiDevice(s.Desc.VirtioScsi.Id, devType, id)
case DISK_DRIVER_PVSCSI:
if s.Desc.PvScsi == nil {
s.Desc.PvScsi = &desc.SGuestPvScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "pvscsi", "scsi"),
}
}
s.Desc.Disks[i].Scsi = desc.NewScsiDevice(s.Desc.PvScsi.Id, devType, id)
case DISK_DRIVER_IDE:
s.Desc.Disks[i].Ide = desc.NewIdeDevice(devType, id)
case DISK_DRIVER_SATA: // -device ahci,id=ahci pci device
s.Desc.Disks[i].Ide = desc.NewIdeDevice(devType, id)
}
}
}
func (s *SKVMGuestInstance) fixDiskDriver() (bool, bool, bool) {
var virtioScsi, pvScsi, sataDisk = false, false, false
isX86 := s.manager.host.IsX8664()
osname := s.GetOsName()
for i := 0; i < len(s.Desc.Disks); i++ {
if !isX86 && (s.Desc.Disks[i].Driver == DISK_DRIVER_IDE ||
s.Desc.Disks[i].Driver == DISK_DRIVER_SATA) {
s.Desc.Disks[i].Driver = DISK_DRIVER_SCSI
} else if osname == OS_NAME_MACOS {
s.Desc.Disks[i].Driver = DISK_DRIVER_SATA
}
switch s.Desc.Disks[i].Driver {
case DISK_DRIVER_SCSI:
virtioScsi = true
case DISK_DRIVER_PVSCSI:
pvScsi = true
case DISK_DRIVER_SATA:
sataDisk = true
}
}
return virtioScsi, pvScsi, sataDisk
}
func (s *SKVMGuestInstance) initVirtioSerial(pciRoot *desc.PCIController) {
s.Desc.VirtioSerial = new(desc.SGuestVirtioSerial)
s.Desc.VirtioSerial.PCIDevice = desc.NewPCIDevice(pciRoot.CType, "virtio-serial-pci", "virtio-serial0")
}
func (s *SKVMGuestInstance) initGuestVga(pciRoot *desc.PCIController) {
var isNotX86 = !s.manager.host.IsX8664()
if s.gpusHasVga() {
s.Desc.Vga = "none"
} else if isNotX86 {
s.Desc.Vga = "virtio-gpu"
} else if s.Desc.Vga == "" {
s.Desc.Vga = "std"
}
s.Desc.VgaDevice = new(desc.SGuestVga)
var vgaDevName string
var options map[string]string
switch s.Desc.Vga {
case "std":
vgaDevName = "VGA"
case "qxl":
vgaDevName = "qxl-vga"
options = map[string]string{
"ram_size": "141557760",
"vram_size": "141557760",
}
case "cirros":
vgaDevName = "cirrus-vga"
case "vmware":
vgaDevName = "vmware-svga"
case "virtio":
vgaDevName = "virtio-vga"
case "virtio-gpu":
vgaDevName = "virtio-gpu-pci"
options = map[string]string{
"max_outputs": "1",
}
case "", "none":
vgaDevName = "none"
}
if vgaDevName != "none" {
s.Desc.VgaDevice.PCIDevice = desc.NewPCIDevice(pciRoot.CType, vgaDevName, "video0")
s.Desc.VgaDevice.PCIDevice.Options = options
}
}
func (s *SKVMGuestInstance) initSpiceDevices(pciRoot *desc.PCIController) {
spice := new(desc.SSpiceDesc)
spice.IntelHDA = &desc.SoundCard{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "intel-hda", "sound0"),
Codec: &desc.Codec{
Id: "sound0-codec0",
Type: "hda-duplex",
Cad: 0,
},
}
var ehciId = "usbspice"
spice.UsbRedirct = &desc.UsbRedirctDesc{
EHCI1: desc.NewUsbController("", -1),
UHCI1: desc.NewUsbController(ehciId, 0),
UHCI2: desc.NewUsbController(ehciId, 2),
UHCI3: desc.NewUsbController(ehciId, 4),
}
spice.UsbRedirct.UsbRedirDev1 = &desc.UsbRedir{
Id: "usbredirdev1",
Source: desc.NewCharDev("spicevmc", "usbredirchardev1", "usbredir"),
}
spice.UsbRedirct.UsbRedirDev2 = &desc.UsbRedir{
Id: "usbredirdev2",
Source: desc.NewCharDev("spicevmc", "usbredirchardev2", "usbredir"),
}
spice.UsbRedirct.EHCI1.PCIDevice = desc.NewPCIDevice(pciRoot.CType, "ich9-usb-ehci1", ehciId)
spice.UsbRedirct.UHCI1.PCIDevice = desc.NewPCIDevice(pciRoot.CType, "ich9-usb-uhci1", "uhci1")
spice.UsbRedirct.UHCI2.PCIDevice = desc.NewPCIDevice(pciRoot.CType, "ich9-usb-uhci2", "uhci2")
spice.UsbRedirct.UHCI3.PCIDevice = desc.NewPCIDevice(pciRoot.CType, "ich9-usb-uhci3", "uhci3")
spice.VdagentSerial = &desc.SGuestVirtioSerial{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "virtio-serial-pci", "vdagent-serial0"),
}
spice.Vdagent = desc.NewCharDev("spicevmc", "vdagent", "vdagent")
spice.VdagentSerialPort = &desc.VirtSerialPort{
Chardev: "vdagent",
Name: "com.redhat.spice.0",
Options: map[string]string{
"nr": "1",
},
}
spice.Options = map[string]string{
"disable-ticketing": "off",
"seamless-migration": "on",
}
s.Desc.VdiDevice.Spice = spice
}
func (s *SKVMGuestInstance) ensureDevicePciAddress(
dev *desc.PCIDevice, function int, multiFunc *bool,
) error {
if function >= 8 {
return errors.Errorf("invalid function %x", function)
} else if function < 0 {
// if function not given, give them default function 0
if dev.PCIAddr != nil {
function = int(dev.Function)
} else {
function = 0
}
}
if dev.PCIAddr != nil {
dev.Function = uint(function)
return s.pciAddrs.Buses[dev.Bus].EnsureSlotFunction(dev.Slot, dev.Function)
}
bus, slot, found := s.findUnusedSlotForController(dev.Controller, function)
if !found {
return errors.Errorf("no valid pci address found ?")
}
s.pciAddrs.Buses[bus].EnsureSlotFunction(uint(slot), uint(function))
dev.PCIAddr = &desc.PCIAddr{
Bus: uint(bus),
Slot: uint(slot),
Function: uint(function),
Multi: multiFunc,
}
return nil
}
func (s *SKVMGuestInstance) findUnusedSlotForController(cont desc.PCI_CONTROLLER_TYPE, function int) (int, int, bool) {
var (
bus, slot = 0, 0
found = false
)
for ; bus < len(s.pciAddrs.Buses); bus++ {
if cont == s.pciAddrs.Buses[bus].Contorller {
slot = s.pciAddrs.Buses[bus].FindNextUnusedSlot(uint(function))
if slot >= 0 {
found = true
break
}
}
}
return bus, slot, found
}
func (s *SKVMGuestInstance) ensurePciAddresses() error {
var err error
if s.Desc.VgaDevice != nil && s.Desc.VgaDevice.PCIDevice != nil {
err = s.ensureDevicePciAddress(s.Desc.VgaDevice.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vga pci address")
}
}
if s.Desc.VirtioSerial != nil {
err = s.ensureDevicePciAddress(s.Desc.VirtioSerial.PCIDevice, -1, nil)
}
if s.Desc.VdiDevice != nil && s.Desc.VdiDevice.Spice != nil {
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.IntelHDA.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vdi hda pci address")
}
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.VdagentSerial.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vdagent serial pci address")
}
multiFunc := true
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIDevice, 0, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb ehci1 pci address")
}
s.Desc.VdiDevice.Spice.UsbRedirct.UHCI2.PCIAddr = s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIAddr.Copy()
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI2.PCIDevice, 1, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb ehci1 pci address")
}
s.Desc.VdiDevice.Spice.UsbRedirct.UHCI3.PCIAddr = s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIAddr.Copy()
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI3.PCIDevice, 2, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb ehci1 pci address")
}
s.Desc.VdiDevice.Spice.UsbRedirct.EHCI1.PCIAddr = s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIAddr.Copy()
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.EHCI1.PCIDevice, 7, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb ehci1 pci address")
}
}
if s.Desc.VirtioScsi != nil {
err = s.ensureDevicePciAddress(s.Desc.VirtioScsi.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure virtio scsi pci address")
}
} else if s.Desc.PvScsi != nil {
err = s.ensureDevicePciAddress(s.Desc.PvScsi.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure pvscsi pci address")
}
}
if s.Desc.SataController != nil {
err = s.ensureDevicePciAddress(s.Desc.SataController.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure SataController pci address")
}
}
// skip pci root or pcie root
for i := 1; i < len(s.Desc.PCIControllers); i++ {
err = s.ensureDevicePciAddress(s.Desc.PCIControllers[i].PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure pci controllers address")
}
}
for i := 0; i < len(s.Desc.Disks); i++ {
if s.Desc.Disks[i].Pci != nil {
err = s.ensureDevicePciAddress(s.Desc.Disks[i].Pci, -1, nil)
if err != nil {
return errors.Wrapf(err, "ensure disk %d pci address", s.Desc.Disks[i].Index)
}
}
}
for i := 0; i < len(s.Desc.Nics); i++ {
if s.Desc.Nics[i].Pci != nil {
err = s.ensureDevicePciAddress(s.Desc.Nics[i].Pci, -1, nil)
if err != nil {
return errors.Wrapf(err, "ensure nic %s pci address", s.Desc.Nics[i].Ifname)
}
}
}
for i := 0; i < len(s.Desc.IsolatedDevices); i++ {
if len(s.Desc.IsolatedDevices[i].VfioDevs) > 0 {
multiFunc := len(s.Desc.IsolatedDevices[i].VfioDevs) > 1
err = s.ensureDevicePciAddress(s.Desc.IsolatedDevices[i].VfioDevs[0].PCIDevice, 0, &multiFunc)
if err != nil {
return errors.Wrapf(err, "ensure isolated device %s pci address", s.Desc.IsolatedDevices[i].VfioDevs[0].PCIAddr)
}
for j := 1; j < len(s.Desc.IsolatedDevices[i].VfioDevs); j++ {
s.Desc.IsolatedDevices[i].VfioDevs[j].PCIAddr = s.Desc.IsolatedDevices[i].VfioDevs[0].PCIAddr.Copy()
err = s.ensureDevicePciAddress(s.Desc.IsolatedDevices[i].VfioDevs[j].PCIDevice, j, nil)
if err != nil {
return errors.Wrapf(err, "ensure isolated device %s pci address", s.Desc.IsolatedDevices[i].VfioDevs[j].PCIAddr)
}
}
}
}
if s.Desc.Usb != nil {
err = s.ensureDevicePciAddress(s.Desc.Usb.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure usb controller pci address")
}
}
if s.Desc.Rng != nil {
err = s.ensureDevicePciAddress(s.Desc.Rng.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure random device pci address")
}
}
anonymousPCIDevs := s.Desc.AnonymousPCIDevs[:0]
for i := 0; i < len(s.Desc.AnonymousPCIDevs); i++ {
if s.isMachineDefaultAddress(s.Desc.AnonymousPCIDevs[i].PCIAddr) {
if _, inUse := s.pciAddrs.IsAddrInUse(s.Desc.AnonymousPCIDevs[i].PCIAddr); inUse {
log.Infof("guest %s anonymous dev addr %s in use", s.GetName(), s.Desc.AnonymousPCIDevs[i].String())
continue
}
}
err = s.ensureDevicePciAddress(s.Desc.AnonymousPCIDevs[i], -1, nil)
if err != nil {
return errors.Wrap(err, "ensure anonymous pci dev pci address")
}
anonymousPCIDevs = append(anonymousPCIDevs, s.Desc.AnonymousPCIDevs[i])
}
if len(anonymousPCIDevs) == 0 {
anonymousPCIDevs = nil
}
s.Desc.AnonymousPCIDevs = anonymousPCIDevs
return nil
}
func (s *SKVMGuestInstance) getNetdevOfThePciAddress(qtree string, addr *desc.PCIAddr) string {
var slotFunc = fmt.Sprintf("addr = %02x.%x", addr.Slot, addr.Function)
var addressFound = false
var lines = strings.Split(strings.TrimSuffix(qtree, "\r\n"), "\\r\\n")
var currentIndentLevel = -1
for _, line := range lines {
trimmedLine := strings.TrimSpace(line)
if trimmedLine == "" {
continue
}
if currentIndentLevel > 0 {
newIndentLevel := len(line) - len(trimmedLine)
if newIndentLevel <= currentIndentLevel {
if addressFound {
break
}
currentIndentLevel = -1
continue
}
if strings.HasPrefix(trimmedLine, slotFunc) {
addressFound = true
continue
}
if strings.HasPrefix(trimmedLine, "netdev =") {
segs := strings.Split(trimmedLine, " ")
netdev := strings.Trim(segs[2], `\\"`)
log.Infof("found netdev %s: %s", netdev, trimmedLine)
return netdev
} else {
continue
}
}
if strings.HasPrefix(trimmedLine, "dev: virtio-net-pci") {
currentIndentLevel = len(line) - len(trimmedLine)
} else {
continue
}
}
return ""
}
// guests description no pci description before host-agent assign pci device address info
// in this case wo need query pci address info by `query-pci` command. Also memory devices.
func (s *SKVMGuestInstance) initGuestDescFromExistingGuest(
cpuList []monitor.HotpluggableCPU, pciInfoList []monitor.PCIInfo,
memoryDevicesInfoList []monitor.MemoryDeviceInfo, memDevs []monitor.Memdev,
scsiNumQueues int64, qtree string,
) error {
if len(pciInfoList) > 1 {
return errors.Errorf("unsupported pci info list with multi bus")
}
unknownDevices := make([]monitor.PCIDeviceInfo, 0)
err := s.initCpuDesc(uint(len(cpuList)))
if err != nil {
return err
}
err = s.initMemDescFromMemoryInfo(memoryDevicesInfoList, memDevs)
if err != nil {
return errors.Wrap(err, "init guest memory devices")
}
s.initMachineDesc()
if s.manager.host.IsX8664() && !s.hasHpet(qtree) {
noHpet := true
s.Desc.NoHpet = &noHpet
}
// This code is designed to ensure compatibility with older guests.
// However, it is not recommended for new guests to generate a desc file from it
pciRoot, _ := s.initGuestPciControllers(false)
err = s.initGuestPciAddresses()
if err != nil {
return errors.Wrap(err, "init guest pci addresses")
}
if err := s.initGuestNetworks(pciRoot, nil); err != nil {
return errors.Wrap(err, "init guest networks")
}
//s.initIsolatedDevices(pciRoot, nil)
s.initQgaDesc()
s.initPvpanicDesc()
s.initIsaSerialDesc()
s.Desc.VdiDevice = new(desc.SGuestVdi)
for i := 0; i < len(pciInfoList[0].Devices); i++ {
pciAddr := &desc.PCIAddr{
Bus: uint(pciInfoList[0].Devices[i].Bus),
Slot: uint(pciInfoList[0].Devices[i].Slot),
Function: uint(pciInfoList[0].Devices[i].Function),
}
switch pciInfoList[0].Devices[i].QdevID {
case "scsi":
_, hasPvScsi, _ := s.fixDiskDriver()
if hasPvScsi && s.Desc.PvScsi == nil {
s.Desc.PvScsi = &desc.SGuestPvScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "pvscsi", "scsi"),
}
} else if s.Desc.VirtioScsi == nil {
s.Desc.VirtioScsi = &desc.SGuestVirtioScsi{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "virtio-scsi-pci", "scsi"),
}
}
if s.Desc.VirtioScsi != nil {
s.Desc.VirtioScsi.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VirtioScsi.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure virtio scsi pci address")
}
if scsiNumQueues > 1 {
numQueues := uint8(scsiNumQueues)
s.Desc.VirtioScsi.NumQueues = &numQueues
}
} else if s.Desc.PvScsi != nil {
s.Desc.PvScsi.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.PvScsi.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure pvscsi pci address")
}
}
case "video0", "video1":
if s.Desc.VgaDevice == nil {
s.initGuestVga(pciRoot)
}
s.Desc.VgaDevice.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VgaDevice.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vga pci address")
}
case "random0":
if s.Desc.Rng == nil {
// in case rng device disable by host options
s.initRandomDevice(pciRoot, true)
}
s.Desc.Rng.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Rng.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure random device pci address")
}
case "usb":
if s.Desc.Usb == nil {
s.initUsbController(pciRoot)
}
s.Desc.Usb.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Usb.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure usb controller pci address")
}
case "sound0":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
s.Desc.VdiDevice.Spice.IntelHDA.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.IntelHDA.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vdi hda pci address")
}
case "vdagent-serial0":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
s.Desc.VdiDevice.Spice.VdagentSerial.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.VdagentSerial.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vdagent serial pci address")
}
case "virtio-serial0":
if s.Desc.VirtioSerial == nil {
s.initVirtioSerial(pciRoot)
}
s.Desc.VirtioSerial.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VirtioSerial.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure virtio serial address")
}
case "usbspice":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
s.Desc.VdiDevice.Spice.UsbRedirct.EHCI1.PCIAddr = pciAddr
multiFunc := true
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.EHCI1.PCIDevice, -1, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb ehci1 pci address")
}
case "uhci1":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
multiFunc := true
s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI1.PCIDevice, -1, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb uhci1 pci address")
}
case "uhci2":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
multiFunc := true
s.Desc.VdiDevice.Spice.UsbRedirct.UHCI2.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI2.PCIDevice, -1, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb uhci2 pci address")
}
case "uhci3":
if s.Desc.VdiDevice.Spice == nil {
s.Desc.Vdi = "spice"
s.initSpiceDevices(pciRoot)
}
multiFunc := true
s.Desc.VdiDevice.Spice.UsbRedirct.UHCI3.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VdiDevice.Spice.UsbRedirct.UHCI3.PCIDevice, -1, &multiFunc)
if err != nil {
return errors.Wrap(err, "ensure vdi usb uhci3 pci address")
}
case "ahci0":
if s.Desc.SataController == nil {
s.Desc.SataController = &desc.SGuestAhciDevice{
PCIDevice: desc.NewPCIDevice(pciRoot.CType, "ahci", "ahci0"),
}
}
s.Desc.SataController.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.SataController.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure SataController pci address")
}
default:
switch {
case strings.HasPrefix(pciInfoList[0].Devices[i].QdevID, "drive_"):
indexStr := strings.TrimPrefix(pciInfoList[0].Devices[i].QdevID, "drive_")
index, err := strconv.Atoi(indexStr)
if err != nil {
log.Errorf("failed parse disk pci id %s", pciInfoList[0].Devices[i].QdevID)
unknownDevices = append(unknownDevices, pciInfoList[0].Devices[i])
continue
}
for i := 0; i < len(s.Desc.Disks); i++ {
if s.Desc.Disks[i].Index == int8(index) {
if s.Desc.Disks[i].Pci == nil {
devType := qemu.GetDiskDeviceModel(DISK_DRIVER_VIRTIO)
s.Desc.Disks[i].Pci = desc.NewPCIDevice(pciRoot.CType, devType, pciInfoList[0].Devices[i].QdevID)
s.Desc.Disks[i].Scsi = nil
s.Desc.Disks[i].Ide = nil
s.Desc.Disks[i].Driver = DISK_DRIVER_VIRTIO
}
s.Desc.Disks[i].Pci.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Disks[i].Pci, -1, nil)
if err != nil {
return errors.Wrapf(err, "ensure disk %d pci address", s.Desc.Disks[i].Index)
}
}
}
case strings.HasPrefix(pciInfoList[0].Devices[i].QdevID, "netdev-"):
ifname := strings.TrimPrefix(pciInfoList[0].Devices[i].QdevID, "netdev-")
for i := 0; i < len(s.Desc.Nics); i++ {
if s.Desc.Nics[i].Ifname == ifname {
s.Desc.Nics[i].Pci.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Nics[i].Pci, -1, nil)
if err != nil {
return errors.Wrapf(err, "ensure nic %s pci address", s.Desc.Nics[i].Ifname)
}
}
}
default:
class := pciInfoList[0].Devices[i].ClassInfo.Class
vendor := pciInfoList[0].Devices[i].ID.Vendor
device := pciInfoList[0].Devices[i].ID.Device
switch { // qemu: docs/specs/pci-ids.txt
case class == 3075 && vendor == 6966 && device == 13: // { 0x0c03, "USB controller", "usb"}, 1b36:000d PCI xhci usb host adapter
if s.Desc.Usb == nil {
s.initUsbController(pciRoot)
}
s.Desc.Usb.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Usb.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure usb controller pci address")
}
case class == 255 && vendor == 6900 && device == 4101: // 0x00ff, 1af4:1005 entropy generator device (legacy)
if s.Desc.Rng == nil {
// in case rng device disable by host options
s.initRandomDevice(pciRoot, true)
}
s.Desc.Rng.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Rng.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure random device pci address")
}
case class == 1920 && vendor == 6900 && device == 4099: // 0x0780, 1af4:1003 console device (legacy)
if s.Desc.VirtioSerial == nil {
s.initVirtioSerial(pciRoot)
}
s.Desc.VirtioSerial.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VirtioSerial.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure virtio serial address")
}
case class == 768 && vendor == 4660 && device == 4369: // { 0x0300, "VGA controller", "display", 0x00ff}, PCI ID: 1234:1111
if s.Desc.VgaDevice == nil {
s.initGuestVga(pciRoot)
}
s.Desc.VgaDevice.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.VgaDevice.PCIDevice, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure vga pci address")
}
case class == 512 && vendor == 6900 && device == 4096: // { 0x0200, "Ethernet controller", "ethernet"}, 1af4:1000 network device (legacy)
// virtio nics has no ids
ifname := s.getNetdevOfThePciAddress(qtree, pciAddr)
index := 0
for ; index < len(s.Desc.Nics); index++ {
if s.Desc.Nics[index].Ifname == ifname {
s.Desc.Nics[index].Pci.PCIAddr = pciAddr
err = s.ensureDevicePciAddress(s.Desc.Nics[index].Pci, -1, nil)
if err != nil {
return errors.Wrapf(err, "ensure nic %s pci address", s.Desc.Nics[index].Ifname)
}
break
}
}
if index >= len(s.Desc.Nics) {
return errors.Errorf("failed find nics ifname")
}
default:
unknownDevices = append(unknownDevices, pciInfoList[0].Devices[i])
}
}
}
}
s.initGuestDisks(pciRoot, nil, true)
for i := 0; i < len(unknownDevices); i++ {
if unknownDevices[i].Bus == 0 && unknownDevices[i].Slot == 0 {
continue // host bridge
}
pciDev := desc.NewPCIDevice(pciRoot.CType, "", unknownDevices[i].QdevID)
pciDev.PCIAddr = &desc.PCIAddr{
Bus: uint(unknownDevices[i].Bus),
Slot: uint(unknownDevices[i].Slot),
Function: uint(unknownDevices[i].Function),
}
err = s.ensureDevicePciAddress(pciDev, -1, nil)
if err != nil {
return errors.Wrap(err, "ensure anonymous pci dev address")
}
if s.isMachineDefaultAddress(pciDev.PCIAddr) {
continue
}
if s.Desc.AnonymousPCIDevs == nil {
s.Desc.AnonymousPCIDevs = make([]*desc.PCIDevice, 0)
}
s.Desc.AnonymousPCIDevs = append(s.Desc.AnonymousPCIDevs, pciDev)
}
return nil
}