fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8333 from rainzm/automated-cherry-pick-of-#8332-upstream-release-3.4

Browse Source 
Automated cherry pick of #8332: feat: support lock storage from selected cachedimage
This commit is contained in:
yunion-ci-robot
2020-10-17 18:23:10 +08:00
committed by GitHub
parent 31de731cbd 7a2c2b8047
commit 41bdfffd57
20 changed files with 157 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/compute/guestdrivers/aliyun.go
View File

@@ -77,8 +77,8 @@ func (self *SAliyunGuestDriver) GetStorageTypes() []string {
}
}
func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SAliyunGuestDriver) GetDetachDiskStatus() ([]string, error) {

4
pkg/compute/guestdrivers/aws.go
View File

@@ -131,8 +131,8 @@ func (self *SAwsGuestDriver) GetStorageTypes() []string {
}
}
func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SAwsGuestDriver) GetDetachDiskStatus() ([]string, error) {

4
pkg/compute/guestdrivers/azure.go
View File

@@ -79,8 +79,8 @@ func (self *SAzureGuestDriver) GetStorageTypes() []string {
}
}
func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SAzureGuestDriver) GetMaxSecurityGroupCount() int {

8
pkg/compute/guestdrivers/baremetals.go
View File

@@ -249,15 +249,15 @@ func (self *SBaremetalGuestDriver) GetStorageTypes() []string {
}
}
func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
if len(storageIds) != 0 {
return models.StorageManager.FetchStorageById(storageIds[0])
return models.StorageManager.FetchStorageById(storageIds[0]), nil
}
bs := host.GetBaremetalstorage()
if bs == nil {
return nil
return nil, nil
}
return bs.GetStorage()
return bs.GetStorage(), nil
}
func (self *SBaremetalGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {

77
pkg/compute/guestdrivers/esxi.go
View File

@@ -79,6 +79,46 @@ func (self *SESXiGuestDriver) GetDefaultSysDiskBackend() string {
return api.STORAGE_LOCAL
}
func (self *SESXiGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
if !options.Options.LockStorageFromCachedimage || len(diskConfig.ImageId) == 0 {
return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
}
var (
image *cloudprovider.SImage
err error
)
obj, err := models.CachedimageManager.FetchById(diskConfig.ImageId)
if err != nil {
return nil, errors.Wrapf(err, "unable to fetch cachedimage %s", diskConfig.ImageId)
}
cachedimage := obj.(*models.SCachedimage)
if len(cachedimage.ExternalId) > 0 || cachedimage.ImageType != cloudprovider.CachedImageTypeSystem {
return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
}
storages, err := cachedimage.GetStorages()
if err != nil {
return nil, errors.Wrapf(err, "unable to GetStorages of cachedimage %s", diskConfig.ImageId)
}
if len(storages) == 0 {
log.Warningf("there no storage associated with cachedimage %q", image.Id)
return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
}
if len(storages) > 1 {
log.Warningf("there are multiple storageCache associated with caheimage %q", image.Id)
}
wantStorageIds := make([]string, len(storages))
for i := range wantStorageIds {
wantStorageIds[i] = storages[i].GetId()
}
for i := range wantStorageIds {
if utils.IsInStringArray(wantStorageIds[i], storageIds) {
log.Infof("use storage %q in where cachedimage %q", wantStorageIds[i], image.Id)
return &storages[i], nil
}
}
return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
}
func (self *SESXiGuestDriver) GetMinimalSysDiskSizeGb() int {
return options.Options.DefaultDiskSizeMB / 1024
}
@@ -183,21 +223,34 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
return desc, errors.Errorf("no such storage cache associated with cacheimage %s", templateId)
}
if len(storageCaches) > 1 {
return desc, errors.Errorf("there are multiple storageCache associated with caheimage '%s' ??!!", templateId)
log.Warningf("there are multiple storageCache associated with caheimage '%s' ??!!", templateId)
}
var hostIp string
storageCacheHost, err := storageCaches[0].GetHost()
if err != nil {
log.Errorf("unable to GetHost of storageCache %s: %v", storageCaches[0].Id, err)
hostIp = storageCaches[0].ExternalId
} else if storageCacheHost == nil {
log.Errorf("unable to GetHost of storageCache %s: result is nil", storageCaches[0].Id)
hostIp = storageCaches[0].ExternalId
} else {
hostIp = storageCacheHost.AccessIp
var storageCacheHost *models.SHost
// select storagecacheHost
for i := range storageCaches {
hosts, err := storageCaches[i].GetHosts()
if err != nil {
return desc, errors.Wrap(err, "storageCaches.GetHosts")
}
for i := range hosts {
if host.GetId() == hosts[i].GetId() {
storageCacheHost = &hosts[i]
}
}
}
hostIp = storageCacheHost.AccessIp
if storageCacheHost == nil {
storageCacheHost, err = storageCaches[0].GetHost()
if err != nil {
return desc, errors.Wrapf(err, "unable to GetHost of storageCache %s", storageCaches[0].Id)
}
if storageCacheHost == nil {
return desc, fmt.Errorf("unable to GetHost of storageCache %s: result is nil", storageCaches[0].Id)
}
}
hostIp := storageCacheHost.AccessIp
imageInfo := SEsxiImageInfo{
ImageType: img.ImageType,
ImageExternalId: img.ExternalId,

4
pkg/compute/guestdrivers/google.go
View File

@@ -81,8 +81,8 @@ func (self *SGoogleGuestDriver) GetStorageTypes() []string {
}
}
func (self *SGoogleGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SGoogleGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SGoogleGuestDriver) GetGuestInitialStateAfterCreate() string {

4
pkg/compute/guestdrivers/huawei.go
View File

@@ -67,8 +67,8 @@ func (self *SHuaweiGuestDriver) GetStorageTypes() []string {
return []string{api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS, api.STORAGE_HUAWEI_SSD}
}
func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SHuaweiGuestDriver) GetDetachDiskStatus() ([]string, error) {

4
pkg/compute/guestdrivers/openstack.go
View File

@@ -91,8 +91,8 @@ func (self *SOpenStackGuestDriver) GetStorageTypes() []string {
return storages
}
func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SOpenStackGuestDriver) GetDetachDiskStatus() ([]string, error) {

4
pkg/compute/guestdrivers/qcloud.go
View File

@@ -75,8 +75,8 @@ func (self *SQcloudGuestDriver) GetStorageTypes() []string {
}
}
func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SQcloudGuestDriver) GetDetachDiskStatus() ([]string, error) {

6
pkg/compute/guestdrivers/virtualization.go
View File

@@ -174,11 +174,11 @@ func (self *SVirtualizedGuestDriver) GetStorageTypes() []string {
return nil
}
func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
if len(storageIds) == 0 {
return host.GetLeastUsedStorage(backend)
return host.GetLeastUsedStorage(diskConfig.Backend), nil
}
return models.StorageManager.FetchStorageById(storageIds[0])
return models.StorageManager.FetchStorageById(storageIds[0]), nil
}
func (self *SVirtualizedGuestDriver) RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *models.SGuest, task taskman.ITask) error {

4
pkg/compute/guestdrivers/zstack.go
View File

@@ -85,8 +85,8 @@ func (self *SZStackGuestDriver) GetMaxSecurityGroupCount() int {
return 1
}
func (self *SZStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
return self.chooseHostStorage(self, host, backend, storageIds)
func (self *SZStackGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SZStackGuestDriver) GetDetachDiskStatus() ([]string, error) {

12
pkg/compute/models/cachedimages.go
View File

@@ -272,6 +272,18 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
}
}
func (image *SCachedimage) GetStorages() ([]SStorage, error) {
sq := StorageManager.Query()
storagecacheimageSubq := StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", image.GetId()).SubQuery()
sq.Join(storagecacheimageSubq, sqlchemy.Equals(sq.Field("storagecache_id"), storagecacheimageSubq.Field("storagecache_id")))
storages := make([]SStorage, 0, 1)
err := db.FetchModelObjects(StorageManager, sq, &storages)
if err != nil {
return nil, errors.Wrap(err, "FetchModelObjects")
}
return storages, nil
}
func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*cloudprovider.SImage, error) {
imgObj, _ := manager.FetchById(imageId)
if imgObj != nil {

2
pkg/compute/models/guestdrivers.go
View File

@@ -73,7 +73,7 @@ type IGuestDriver interface {
GetRandomNetworkTypes() []string
GetStorageTypes() []string
ChooseHostStorage(host *SHost, backend string, storageIds []string) *SStorage
ChooseHostStorage(host *SHost, diskConfig *api.DiskConfig, storageIds []string) (*SStorage, error)
StartGuestCreateTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error

21
pkg/compute/models/guests.go
View File

@@ -3436,11 +3436,11 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
return disk, nil
}
func (self *SGuest) ChooseHostStorage(host *SHost, backend string, candidate *schedapi.CandidateDisk) *SStorage {
func (self *SGuest) ChooseHostStorage(host *SHost, diskConfig *api.DiskConfig, candidate *schedapi.CandidateDisk) (*SStorage, error) {
if candidate == nil || len(candidate.StorageIds) == 0 {
return self.GetDriver().ChooseHostStorage(host, backend, nil)
return self.GetDriver().ChooseHostStorage(host, diskConfig, nil)
}
return self.GetDriver().ChooseHostStorage(host, backend, candidate.StorageIds)
return self.GetDriver().ChooseHostStorage(host, diskConfig, candidate.StorageIds)
}
func (self *SGuest) createDiskOnHost(
@@ -3455,7 +3455,10 @@ func (self *SGuest) createDiskOnHost(
backupCandidate *schedapi.CandidateDisk,
autoAttach bool,
) (*SDisk, error) {
var storage *SStorage
var (
storage *SStorage
err error
)
if len(diskConfig.Storage) > 0 {
_storage, err := StorageManager.FetchByIdOrName(userCred, diskConfig.Storage)
if err != nil {
@@ -3466,7 +3469,10 @@ func (self *SGuest) createDiskOnHost(
}
storage = _storage.(*SStorage)
} else {
storage = self.ChooseHostStorage(host, diskConfig.Backend, candidate)
storage, err = self.ChooseHostStorage(host, diskConfig, candidate)
if err != nil {
return nil, errors.Wrap(err, "ChooseHostStorage")
}
}
if storage == nil {
return nil, fmt.Errorf("No storage on %s to create disk for %s", host.GetName(), diskConfig.Backend)
@@ -3478,7 +3484,10 @@ func (self *SGuest) createDiskOnHost(
}
if len(self.BackupHostId) > 0 {
backupHost := HostManager.FetchHostById(self.BackupHostId)
backupStorage := self.ChooseHostStorage(backupHost, diskConfig.Backend, backupCandidate)
backupStorage, err := self.ChooseHostStorage(backupHost, diskConfig, backupCandidate)
if err != nil {
return nil, errors.Wrap(err, "ChooseHostStorage")
}
diff, err := db.Update(disk, func() error {
disk.BackupStorageId = backupStorage.Id
return nil

5
pkg/compute/models/helper.go
View File

@@ -100,7 +100,10 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred
return nil, err
}
defaultStorage := GetDriver(hypervisor).ChooseHostStorage(baremetal, "", nil)
defaultStorage, err := GetDriver(hypervisor).ChooseHostStorage(baremetal, &api.DiskConfig{}, nil)
if err != nil {
return nil, errors.Wrap(err, "ChooseHostStorage")
}
if defaultStorage == nil {
return nil, httperrors.NewInsufficientResourceError("no valid storage on host")
}

12
pkg/compute/models/storagecaches.go
View File

@@ -133,7 +133,7 @@ func (self *SStoragecache) GetRegion() (*SCloudregion, error) {
return region, nil
}
func (self *SStoragecache) getHostId() (string, error) {
func (self *SStoragecache) GetHosts() ([]SHost, error) {
hoststorages := HoststorageManager.Query().SubQuery()
storages := StorageManager.Query().SubQuery()
@@ -153,7 +153,15 @@ func (self *SStoragecache) getHostId() (string, error) {
sqlchemy.IsTrue(storages.Field("enabled")))).
Filter(sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id"))).All(&hosts)
if err != nil {
return "", err
return nil, err
}
return hosts, nil
}
func (self *SStoragecache) getHostId() (string, error) {
hosts, err := self.GetHosts()
if err != nil {
return "", errors.Wrap(err, "GetHosts")
}
hostIds := make([]string, 0)

2
pkg/compute/options/options.go
View File

@@ -146,6 +146,8 @@ type ComputeOptions struct {
SyncStorageCapacityUsedIntervalMinutes int `help:"interval sync storage capacity used" default:"10"`
LockStorageFromCachedimage bool `help:"must use storage in where selected cachedimage when creating vm"`
SCapabilityOptions
SASControllerOptions
common_options.CommonOptions

7
pkg/compute/tasks/guest_backup_tasks.go
View File

@@ -328,7 +328,12 @@ func (self *GuestCreateBackupTask) StartCreateBackupDisks(ctx context.Context, g
if len(candidateDisks) >= i {
candidateDisk = candidateDisks[i]
}
storage := guest.ChooseHostStorage(host, api.STORAGE_LOCAL, candidateDisk)
diskConfig := &api.DiskConfig{Backend: api.STORAGE_LOCAL}
storage, err := guest.ChooseHostStorage(host, diskConfig, candidateDisk)
if err != nil {
self.TaskFailed(ctx, guest, jsonutils.NewString(fmt.Sprintf("unable to ChooseHostStorage: %v", err)))
return
}
if storage == nil {
self.TaskFailed(ctx, guest, jsonutils.NewString("Get backup storage error"))
return

22
pkg/hostman/storageman/storage_agent.go
View File

@@ -299,13 +299,22 @@ func (as *SAgentStorage) AgentDeployGuest(ctx context.Context, data interface{})
},
VddkInfo: &vddkInfo,
})
customize := false
if err != nil {
log.Errorf("DeployClient.DeployGuestFs: %s", err)
// if deploy fail, try customization
log.Errorf("unable to DeployGuestFs: %v", err)
customize = true
} else if deploy == nil {
log.Errorf("unable to DeployGuestFs: deploy is nil")
customize = true
} else if len(deploy.Os) == 0 {
log.Errorf("unable to DeployGuestFs: os is empty")
customize = true
}
if customize == true {
as.waitVmToolsVersion(ctx, vm)
err = vm.DoCustomize(ctx, desc)
if err != nil {
return nil, errors.Wrap(err, "VM.DoCustomize")
log.Errorf("unable to DoCustomize for vm %s: %v", vm.GetId(), err)
}
}
@@ -327,7 +336,12 @@ func (as *SAgentStorage) AgentDeployGuest(ctx context.Context, data interface{})
updated.Add(array, "disks")
updated.Add(jsonutils.NewString(vm.GetGlobalId()), "uuid")
updated.Add(jsonutils.NewString(realHost.GetAccessIp()), "host_ip")
ret := jsonutils.Marshal(deploy)
var ret jsonutils.JSONObject
if deploy != nil {
ret = jsonutils.Marshal(deploy)
} else {
ret = jsonutils.NewDict()
}
ret.(*jsonutils.JSONDict).Update(updated)
return ret, nil
}

1
pkg/multicloud/esxi/virtualmachine.go
View File

@@ -1147,6 +1147,7 @@ func (self *SVirtualMachine) DoCustomize(ctx context.Context, params jsonutils.J
}
spec.Identity = &sysPrep
}
log.Infof("customize spec: %#v", spec)
task, err := self.getVmObj().Customize(ctx, *spec)
if err != nil {
return errors.Wrap(err, "object.VirtualMachine.Customize")