Merge pull request #8686 from rainzm/fix/GetIImageById

Fix for Esxi and GetIImage

pkg/compute/models/cachedimages.go

@@ -288,7 +288,8 @@ func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred m
 	imgObj, _ := manager.FetchById(imageId)
 	if imgObj != nil {
 		cachedImage := imgObj.(*SCachedimage)
-		if !refresh && cachedImage.GetStatus() == cloudprovider.IMAGE_STATUS_ACTIVE && len(cachedImage.GetOSType()) > 0 && !cachedImage.isRefreshSessionExpire() {
+		oSTypeOk := options.Options.NoCheckOsTypeForCachedImage || len(cachedImage.GetOSType()) > 0
+		if !refresh && cachedImage.GetStatus() == cloudprovider.IMAGE_STATUS_ACTIVE && oSTypeOk && !cachedImage.isRefreshSessionExpire() {
 			return cachedImage.GetImage()
 		} else if len(cachedImage.ExternalId) > 0 { // external image, request refresh
 			return cachedImage.requestRefreshExternalImage(ctx, userCred)

pkg/compute/models/storages.go

@@ -762,6 +762,61 @@ var CapacityUsedCloudStorageProvider = []string{
 	api.CLOUD_PROVIDER_VMWARE,
 }
 
+func (sm *SStorageManager) SyncCapacityUsedForEsxiStorage(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	cpQ := CloudproviderManager.Query("id").Equals("provider", api.CLOUD_PROVIDER_VMWARE)
+	cloudproviders := make([]SCloudprovider, 0)
+	err := db.FetchModelObjects(CloudproviderManager, cpQ, &cloudproviders)
+	if err != nil {
+		log.Errorf("unable to FetchModelObjects: %v", err)
+	}
+	for i := range cloudproviders {
+		cp := cloudproviders[i]
+		icp, err := cp.GetProvider()
+		if err != nil {
+			log.Errorf("unable to GetProvider: %v", err)
+			continue
+		}
+		iregion, err := icp.GetOnPremiseIRegion()
+		if err != nil {
+			log.Errorf("unable to GetOnPremiseIRegion: %v", err)
+			continue
+		}
+		css, err := iregion.GetIStorages()
+		if err != nil {
+			log.Errorf("unable to GetIStorages: %v", err)
+			continue
+		}
+		storageSizeMap := make(map[string]int64, len(css))
+		for i := range css {
+			id := css[i].GetGlobalId()
+			size := css[i].GetCapacityUsedMB()
+			storageSizeMap[id] = size
+		}
+		sQ := sm.Query().Equals("manager_id", cp.GetId())
+		storages := make([]SStorage, 0, 5)
+		err = db.FetchModelObjects(sm, sQ, &storages)
+		if err != nil {
+			log.Errorf("unable to fetch storages with sql %q: %v", sQ.String(), err)
+			continue
+		}
+		for i := range storages {
+			s := &storages[i]
+			newSize, ok := storageSizeMap[s.GetExternalId()]
+			if !ok {
+				log.Warningf("can't find usedSize for storage %q", s.GetId())
+				continue
+			}
+			_, err = db.Update(s, func() error {
+				s.ActualCapacityUsed = newSize
+				return nil
+			})
+			if err != nil {
+				log.Errorf("unable to udpate storage %q: %v", s.GetId(), err)
+			}
+		}
+	}
+}
+
 func (sm *SStorageManager) SyncCapacityUsedForStorage(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 	cpSubQ := CloudproviderManager.Query("id").In("provider", CapacityUsedCloudStorageProvider).SubQuery()
 	sQ := sm.Query()

pkg/compute/options/options.go

@@ -145,7 +145,7 @@ type ComputeOptions struct {
 
 	EnableAutoRenameProject bool `help:"when it set true, auto create project will rename when cloud project name changed" default:"false"`
 
-	SyncStorageCapacityUsedIntervalMinutes int  `help:"interval sync storage capacity used" default:"10"`
+	SyncStorageCapacityUsedIntervalMinutes int  `help:"interval sync storage capacity used" default:"20"`
 	LockStorageFromCachedimage             bool `help:"must use storage in where selected cachedimage when creating vm"`
 
 	SyncExtDiskSnapshotIntervalMinutes int `help:"sync snapshot for external disk" default:"20"`
@@ -159,6 +159,8 @@ type ComputeOptions struct {
 
 	DefaultNetworkGatewayAddressEsxi uint32 `help:"Default address for network gateway" default:"1"`
 
+	NoCheckOsTypeForCachedImage bool `help:"Don't check os type for cached image"`
+
 	esxi.EsxiOptions
 }
 

pkg/compute/service/service.go

@@ -141,7 +141,7 @@ func StartService() {
 		cron.AddJobAtIntervalsWithStartRun("AutoSyncCloudaccountTask", time.Duration(opts.CloudAutoSyncIntervalSeconds)*time.Second, models.CloudaccountManager.AutoSyncCloudaccountTask, true)
 		cron.AddJobAtIntervalsWithStartRun("ReconcileBackupGuests", time.Duration(opts.ReconcileGuestBackupIntervalSeconds)*time.Second, models.GuestManager.ReconcileBackupGuests, true)
 
-		cron.AddJobAtIntervalsWithStartRun("SyncCapacityUsedForStorage", time.Duration(opts.SyncStorageCapacityUsedIntervalMinutes)*time.Minute, models.StorageManager.SyncCapacityUsedForStorage, true)
+		cron.AddJobAtIntervalsWithStartRun("SyncCapacityUsedForEsxiStorage", time.Duration(opts.SyncStorageCapacityUsedIntervalMinutes)*time.Minute, models.StorageManager.SyncCapacityUsedForEsxiStorage, true)
 
 		cron.AddJobAtIntervalsWithStartRun("AutoSyncExtDiskSnapshot", time.Duration(opts.SyncExtDiskSnapshotIntervalMinutes)*time.Minute, models.DiskManager.AutoSyncExtDiskSnapshot, true)
 

pkg/compute/tasks/guest_create_task.go

@@ -110,10 +110,10 @@ func (self *GuestCreateTask) StartDeployGuest(ctx context.Context, guest *models
 func (self *GuestCreateTask) OnDeployGuestDescComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	guest := obj.(*models.SGuest)
 	// sync capacityUsed for storage
-	err := guest.SyncCapacityUsedForStorage(ctx, nil)
-	if err != nil {
-		log.Errorf("unable to SyncCapacityUsedForStorage: %v", err)
-	}
+	// err := guest.SyncCapacityUsedForStorage(ctx, nil)
+	// if err != nil {
+	// 	log.Errorf("unable to SyncCapacityUsedForStorage: %v", err)
+	// }
 
 	// bind eip
 	{

pkg/compute/tasks/guest_delete_task.go

@@ -317,17 +317,17 @@ func (self *GuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.I
 	guest.DeleteEip(ctx, self.UserCred)
 	guest.GetDriver().OnDeleteGuestFinalCleanup(ctx, guest, self.UserCred)
 	// sync capacity used for storage
-	ja, err := self.Params.GetArray(STORAGEIDS)
-	if err == nil {
-		storageIds := make([]string, len(ja))
-		for i := range ja {
-			storageIds[i], _ = ja[i].GetString()
-		}
-		err = guest.SyncCapacityUsedForStorage(ctx, storageIds)
-		if err != nil {
-			log.Errorf("unable to SyncCapacityUsedForStoarage: %v", err)
-		}
-	}
+	// ja, err := self.Params.GetArray(STORAGEIDS)
+	// if err == nil {
+	// 	storageIds := make([]string, len(ja))
+	// 	for i := range ja {
+	// 		storageIds[i], _ = ja[i].GetString()
+	// 	}
+	// 	err = guest.SyncCapacityUsedForStorage(ctx, storageIds)
+	// 	if err != nil {
+	// 		log.Errorf("unable to SyncCapacityUsedForStoarage: %v", err)
+	// 	}
+	// }
 	self.DeleteGuest(ctx, guest)
 }
 

pkg/multicloud/esxi/host.go

@@ -958,7 +958,9 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat
 func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SDatastore, params SCreateVMParam) (*SVirtualMachine, error) {
 	ovm := from.getVmObj()
 
-	deviceChange := make([]types.BaseVirtualDeviceConfigSpec, 0, 5)
+	deviceChange := make([]types.BaseVirtualDeviceConfigSpec, 0, 3)
+
+	addDeviceChange := make([]types.BaseVirtualDeviceConfigSpec, 0, 3)
 
 	// change nic if set
 	if params.Nics != nil && len(params.Nics) > 0 {
@@ -994,11 +996,16 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 				op = types.VirtualDeviceConfigSpecOperationEdit
 				host.changeNic(originNics[nicIndex], dev)
 				dev = originNics[nicIndex]
+				deviceChange = append(deviceChange, &types.VirtualDeviceConfigSpec{
+					Operation: op,
+					Device:    dev,
+				})
+			} else {
+				addDeviceChange = append(addDeviceChange, &types.VirtualDeviceConfigSpec{
+					Operation: op,
+					Device:    dev,
+				})
 			}
-			deviceChange = append(deviceChange, &types.VirtualDeviceConfigSpec{
-				Operation: op,
-				Device:    dev,
-			})
 		}
 	}
 
@@ -1015,7 +1022,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 				if host.isVersion50() {
 					driver = "scsi"
 				}
-				deviceChange = append(deviceChange, addDevSpec(NewSCSIDev(key, 100, driver)))
+				addDeviceChange = append(deviceChange, addDevSpec(NewSCSIDev(key, 100, driver)))
 			}
 		} else {
 			ideDevs, err := from.FindController(ctx, "ide")
@@ -1024,8 +1031,8 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 			}
 			if len(ideDevs) == 0 {
 				// add ide driver
-				deviceChange = append(deviceChange, addDevSpec(NewIDEDev(200, 0)))
-				deviceChange = append(deviceChange, addDevSpec(NewIDEDev(200, 1)))
+				addDeviceChange = append(deviceChange, addDevSpec(NewIDEDev(200, 0)))
+				addDeviceChange = append(deviceChange, addDevSpec(NewIDEDev(200, 1)))
 			}
 		}
 	}
@@ -1093,7 +1100,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 		return nil, errors.Error("clone successfully but unable to NewVirtualMachine")
 	}
 
-	deviceChange = make([]types.BaseVirtualDeviceConfigSpec, 0, 5)
+	deviceChange = addDeviceChange
 	// adjust disk
 	var i int
 	if len(params.Disks) > 0 {

pkg/multicloud/esxi/storage.go

@@ -270,6 +270,44 @@ func (self *SDatastore) FetchTemplateVMs() ([]*SVirtualMachine, error) {
 	return self.datacenter.fetchVMsWithFilter(filter)
 }
 
+func (self *SDatastore) FetchTemplateVMById(id string) (*SVirtualMachine, error) {
+	mods := self.getDatastore()
+	filter := property.Filter{}
+	uuid := toTemplateUuid(id)
+	filter["summary.config.uuid"] = uuid
+	filter["config.template"] = true
+	filter["datastore"] = mods.Reference()
+	vms, err := self.datacenter.fetchVMsWithFilter(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
+}
+
+func (self *SDatastore) FetchFakeTempateVMById(id string, regex string) (*SVirtualMachine, error) {
+	mods := self.getDatastore()
+	filter := property.Filter{}
+	uuid := toTemplateUuid(id)
+	filter["summary.config.uuid"] = uuid
+	filter["datastore"] = mods.Reference()
+	filter["summary.runtime.powerState"] = types.VirtualMachinePowerStatePoweredOff
+	movms, err := self.datacenter.fetchMoVms(filter, []string{"name"})
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to fetch mo.VirtualMachines")
+	}
+	vms, err := self.datacenter.fetchFakeTemplateVMs(movms, regex)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
+}
+
 func (self *SDatastore) FetchFakeTempateVMs(regex string) ([]*SVirtualMachine, error) {
 	mods := self.getDatastore()
 	filter := property.Filter{}

pkg/multicloud/esxi/storagecache.go

@@ -106,32 +106,9 @@ func (self *SDatastoreImageCache) getFakeTempateVMs() ([]*SVirtualMachine, error
 	return self.datastore.FetchFakeTempateVMs("")
 }
 
-func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, error) {
+func (self *SDatastoreImageCache) GetIImageInImagecache() ([]cloudprovider.ICloudImage, error) {
 	ctx := context.Background()
 	ret := make([]cloudprovider.ICloudImage, 0, 2)
-	log.Infof("start to GetIImages")
-
-	realTemplates, err := self.getTempalteVMs()
-	if err != nil {
-		return nil, errors.Wrap(err, "getTemplateVMs")
-	}
-	fakeTemplates, err := self.getFakeTempateVMs()
-	if err != nil {
-		return nil, errors.Wrap(err, "getFakeTempateVMs")
-	}
-
-	for i := range realTemplates {
-		ret = append(ret, NewVMTemplate(realTemplates[i], self))
-	}
-	for i := range fakeTemplates {
-		ret = append(ret, NewVMTemplate(fakeTemplates[i], self))
-	}
-
-	log.Errorf("get templates successfully")
-	for i := range fakeTemplates {
-		log.Infof("fake template name: %s", fakeTemplates[i].GetName())
-	}
-
 	files, err := self.datastore.ListDir(ctx, IMAGE_CACHE_DIR_NAME)
 	if errors.Cause(err) == errors.ErrNotFound {
 		return ret, nil
@@ -177,8 +154,74 @@ func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, err
 	return ret, nil
 }
 
+func (self *SDatastoreImageCache) GetIImageInTemplateVMs() ([]cloudprovider.ICloudImage, error) {
+	ret := make([]cloudprovider.ICloudImage, 0, 2)
+	log.Infof("start to GetIImages")
+
+	realTemplates, err := self.getTempalteVMs()
+	if err != nil {
+		return nil, errors.Wrap(err, "getTemplateVMs")
+	}
+	fakeTemplates, err := self.getFakeTempateVMs()
+	if err != nil {
+		return nil, errors.Wrap(err, "getFakeTempateVMs")
+	}
+
+	for i := range realTemplates {
+		ret = append(ret, NewVMTemplate(realTemplates[i], self))
+	}
+	for i := range fakeTemplates {
+		ret = append(ret, NewVMTemplate(fakeTemplates[i], self))
+	}
+
+	log.Infof("get templates successfully")
+	for i := range fakeTemplates {
+		log.Infof("fake template name: %s", fakeTemplates[i].GetName())
+	}
+	return ret, nil
+}
+
+func (self *SDatastoreImageCache) GetIImageInTemplateVMsById(id string) (cloudprovider.ICloudImage, error) {
+	if tempalteNameRegex != nil {
+		vm, err := self.datastore.FetchFakeTempateVMById(id, "")
+		log.Infof("FetchFakeTempateVMById: %v, %v", vm, err)
+		if err == nil {
+			return NewVMTemplate(vm, self), nil
+		}
+		if errors.Cause(err) != errors.ErrNotFound {
+			return nil, err
+		}
+	}
+	vm, err := self.datastore.FetchTemplateVMById(id)
+	if err == nil {
+		return NewVMTemplate(vm, self), nil
+	}
+	return nil, err
+}
+
+func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, error) {
+	images1, err := self.GetIImageInTemplateVMs()
+	if err != nil {
+		return nil, err
+	}
+	images2, err := self.GetIImageInImagecache()
+	if err != nil {
+		return nil, err
+	}
+	return append(images1, images2...), nil
+}
+
 func (self *SDatastoreImageCache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) {
-	images, err := self.GetIImages()
+	// check templatevms
+	image, err := self.GetIImageInTemplateVMsById(extId)
+	if err == nil {
+		return image, nil
+	}
+	if errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	log.Infof("start to GetIImageInImagecache")
+	images, err := self.GetIImageInImagecache()
 	if err != nil {
 		return nil, err
 	}

pkg/multicloud/esxi/virtualmachine.go

@@ -1168,7 +1168,7 @@ func (self *SVirtualMachine) DoCustomize(ctx context.Context, params jsonutils.J
 	spec.NicSettingMap = maps
 
 	var (
-		osName = "Linux"
+		osName string
 		name   = "yunionhost"
 	)
 	if params.Contains("os_name") {
@@ -1184,7 +1184,7 @@ func (self *SVirtualMachine) DoCustomize(ctx context.Context, params jsonutils.J
 			TimeZone: "Asia/Shanghai",
 		}
 		spec.Identity = &linuxPrep
-	} else {
+	} else if osName == "Windows" {
 		sysPrep := types.CustomizationSysprep{
 			GuiUnattended: types.CustomizationGuiUnattended{
 				TimeZone:  210,