Merge pull request #7907 from wanyaoqi/automated-cherry-pick-of-#7905-upstream-release-3.3

Automated cherry pick of #7905: webconsole: fix defer order to release zombie process

pkg/apis/compute/cachedimage.go

@@ -27,7 +27,7 @@ type CachedImageUncacheImageInput struct {
 }
 
 type CachedimageDetails struct {
-	apis.StandaloneResourceDetails
+	apis.SharableVirtualResourceDetails
 
 	SCachedimage
 

pkg/apis/compute/cachedimages_const.go

@@ -18,7 +18,7 @@ const (
 	CACHED_IMAGE_STATUS_INIT         = "init"
 	CACHED_IMAGE_STATUS_SAVING       = "saving"
 	CACHED_IMAGE_STATUS_CACHING      = "caching"
-	CACHED_IMAGE_STATUS_READY        = "ready"
+	CACHED_IMAGE_STATUS_ACTIVE       = "active"
 	CACHED_IMAGE_STATUS_DELETING     = "deleting"
 	CACHED_IMAGE_STATUS_CACHE_FAILED = "cache_fail"
 

pkg/apis/compute/cloudregions_const.go

@@ -57,6 +57,7 @@ const (
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
 	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
+	CITY_WU_LAN_CHA_BU = "Wulanchabu"  //乌兰察布
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京

pkg/apis/compute/input.go

@@ -49,7 +49,7 @@ type KeypairListInput struct {
 }
 
 type CachedimageListInput struct {
-	apis.StandaloneResourceListInput
+	apis.SharableVirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput

pkg/apis/compute/secgroup.go

@@ -248,3 +248,6 @@ type GuestsecgroupDetails struct {
 	// 安全组名称
 	Secgroup string `json:"secgroup"`
 }
+
+type SecurityGroupPurgeInput struct {
+}

pkg/baremetal/manager.go

@@ -61,6 +61,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/dhcp"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
@@ -1398,7 +1399,15 @@ func (b *SBaremetalInstance) DoRedfishPowerOn() error {
 	b.ClearSSHConfig()
 	redfishApi := b.GetRedfishCli(ctx)
 	if redfishApi != nil {
-		return redfishApi.Reset(ctx, "On")
+		err := redfishApi.Reset(ctx, "On")
+		if err != nil {
+			if httputils.ErrorCode(err) == 409 {
+				log.Warningf("redfishApi.Reset On fail %s", err)
+			} else {
+				return errors.Wrap(err, "redfishApi.Reset On")
+			}
+		}
+		return nil
 	}
 	return fmt.Errorf("Baremetal %s redfishApi is nil", b.GetId())
 }

pkg/cloudcommon/notifyclient/notify.go

@@ -126,7 +126,7 @@ func RawNotify(recipientId []string, isGroup bool, channel notify.TNotifyChannel
 	msg.Topic = topic
 	body, _ := getContent(event, "content", channel, data)
 	if len(body) == 0 {
-		body = data.String()
+		body, _ = data.GetString()
 	}
 	msg.Msg = body
 	// log.Debugf("send notification %s %s", topic, body)

pkg/cloudprovider/resources.go

@@ -156,7 +156,7 @@ type ICloudZone interface {
 }
 
 type ICloudImage interface {
-	ICloudResource
+	IVirtualResource
 
 	Delete(ctx context.Context) error
 	GetIStoragecache() ICloudStoragecache
@@ -173,6 +173,7 @@ type ICloudImage interface {
 	GetImageFormat() string
 	GetCreatedAt() time.Time
 	UEFI() bool
+	GetPublicScope() rbacutils.TRbacScope
 }
 
 type ICloudStoragecache interface {

pkg/compute/guestdrivers/base.go

@@ -314,6 +314,10 @@ func (self *SBaseGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
 	return false
 }
 
+func (self *SBaseGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return true
+}
+
 func (self *SBaseGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_CONFIG
 }

pkg/compute/guestdrivers/esxi.go

@@ -172,7 +172,7 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 	if img.ImageType != cloudprovider.CachedImageTypeSystem {
 		return desc
 	}
-	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_READY).SubQuery()
+	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE).SubQuery()
 	scQ := models.StoragecacheManager.Query().In("id", sciSubQ)
 	storageCaches := make([]models.SStoragecache, 0, 1)
 	err = db.FetchModelObjects(models.StoragecacheManager, scQ, &storageCaches)
@@ -368,3 +368,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/guestdrivers/qcloud.go

@@ -226,6 +226,10 @@ func (self *SQcloudGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_RUNNING
 }
 
+func (self *SQcloudGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return false
+}
+
 func (self *SQcloudGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }

pkg/compute/hostdrivers/managedvirtual.go

@@ -182,7 +182,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		osType, _ := params.GetString("properties", "os_type")
 
 		scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId, "")
-		if scimg.Status != api.CACHED_IMAGE_STATUS_READY {
+		if scimg.Status != api.CACHED_IMAGE_STATUS_ACTIVE {
 			scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host")
 		}
 		iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, "")
@@ -208,7 +208,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		if err := snapshot.Delete(); err != nil {
 			log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err)
 		}
-		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_READY, "")
+		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_ACTIVE, "")
 		return result, nil
 	})
 	return nil

pkg/compute/models/cachedimages.go

@@ -37,11 +37,12 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 type SCachedimageManager struct {
-	db.SStandaloneResourceBaseManager
+	db.SSharableVirtualResourceBaseManager
 	db.SExternalizedResourceBaseManager
 }
 
@@ -49,7 +50,7 @@ var CachedimageManager *SCachedimageManager
 
 func init() {
 	CachedimageManager = &SCachedimageManager{
-		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
 			SCachedimage{},
 			"cachedimages_tbl",
 			"cachedimage",
@@ -60,7 +61,7 @@ func init() {
 }
 
 type SCachedimage struct {
-	db.SStandaloneResourceBase
+	db.SSharableVirtualResourceBase
 	db.SExternalizedResourceBase
 
 	// 镜像大小单位: Byte
@@ -100,10 +101,10 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {
 	if cnt > 0 {
 		return httperrors.NewNotEmptyError("The image has been cached on storages")
 	}
-	if self.GetStatus() == "active" && !self.isReferenceSessionExpire() {
+	if self.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !self.isReferenceSessionExpire() {
 		return httperrors.NewConflictError("the image reference session has not been expired!")
 	}
-	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
 func (self *SCachedimage) isReferenceSessionExpire() bool {
@@ -194,32 +195,44 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
-	imgId, _ := info.GetString("id")
-	if len(imgId) == 0 {
+	img := struct {
+		Id          string
+		Size        int64
+		Name        string
+		Status      string
+		IsPublic    bool
+		ProjectId   string `json:"tenant_id"`
+		PublicScope string
+	}{}
+	err := info.Unmarshal(&img)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Unmarshal %s", info.String())
+	}
+	if len(img.Id) == 0 {
 		return nil, fmt.Errorf("invalid image info")
 	}
+	if len(img.Name) == 0 {
+		img.Name = img.Id
+	}
 
 	imageCache := SCachedimage{}
 	imageCache.SetModelManager(manager, &imageCache)
 
-	size, _ := info.Int("size")
-	name, _ := info.GetString("name")
-	if len(name) == 0 {
-		name = imgId
-	}
-
-	name, err := db.GenerateName(manager, nil, name)
+	img.Name, err = db.GenerateName(manager, nil, img.Name)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "db.GenerateName(%s)", img.Name)
 	}
 
-	err = manager.RawQuery().Equals("id", imgId).First(&imageCache)
+	err = manager.RawQuery().Equals("id", img.Id).First(&imageCache)
 	if err != nil {
 		if err == sql.ErrNoRows { // insert
-			imageCache.Id = imgId
-			imageCache.Name = name
-			imageCache.Size = size
+			imageCache.Id = img.Id
+			imageCache.Name = img.Name
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 
 			err = manager.TableSpec().Insert(ctx, &imageCache)
@@ -230,13 +243,16 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 
 			return &imageCache, nil
 		} else {
-			log.Errorf("fetching image cache (%s) failed: %s", imgId, err)
+			log.Errorf("fetching image cache (%s) failed: %s", img.Id, err)
 			return nil, err
 		}
 	} else { // update
 		diff, err := db.Update(&imageCache, func() error {
-			imageCache.Size = size
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 			if imageCache.Deleted == true {
 				imageCache.Deleted = false
@@ -323,16 +339,15 @@ func (manager *SCachedimageManager) FetchCustomizeColumns(
 	isList bool,
 ) []api.CachedimageDetails {
 	rows := make([]api.CachedimageDetails, len(objs))
-	stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	virtRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range rows {
 		ci := objs[i].(*SCachedimage)
 		rows[i] = api.CachedimageDetails{
-			StandaloneResourceDetails: stdRows[i],
-			Status:                    ci.GetStatus(),
-			OsType:                    ci.GetOSType(),
-			OsDistribution:            ci.GetOSDistribution(),
-			OsVersion:                 ci.GetOSVersion(),
-			Hypervisor:                ci.GetHypervisor(),
+			SharableVirtualResourceDetails: virtRows[i],
+			OsType:                         ci.GetOSType(),
+			OsDistribution:                 ci.GetOSDistribution(),
+			OsVersion:                      ci.GetOSVersion(),
+			Hypervisor:                     ci.GetHypervisor(),
 		}
 		rows[i].CachedCount, _ = ci.getStoragecacheCount()
 	}
@@ -372,7 +387,7 @@ func (self *SCachedimage) PerformUncacheImage(ctx context.Context, userCred mccl
 }
 
 func (self *SCachedimage) addRefCount() {
-	if self.GetStatus() != "active" {
+	if self.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE {
 		return
 	}
 	_, err := db.Update(self, func() error {
@@ -397,7 +412,7 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 		Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))).
 		Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)).
-		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_READY)).
+		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE)).
 		Filter(sqlchemy.Equals(host.Field("status"), api.HOST_STATUS_RUNNING)).
 		Filter(sqlchemy.IsTrue(host.Field("enabled"))).
 		Filter(sqlchemy.Equals(host.Field("host_status"), api.HOST_ONLINE)).
@@ -455,12 +470,17 @@ func (self *SCachedimage) canDeleteLastCache() bool {
 	return false
 }
 
-func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error {
+func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = image.GetName()
 		self.Size = image.GetSizeByte()
 		self.ExternalId = image.GetGlobalId()
 		self.ImageType = image.GetImageType()
+		self.PublicScope = string(image.GetPublicScope())
+		self.Status = image.GetStatus()
+		if image.GetPublicScope() == rbacutils.ScopeSystem {
+			self.IsPublic = true
+		}
 		self.UEFI = tristate.NewFromBool(image.UEFI())
 		sImage := cloudprovider.CloudImage2Image(image)
 		self.Info = jsonutils.Marshal(&sImage)
@@ -468,10 +488,12 @@ func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mccli
 		return nil
 	})
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
+
+	SyncCloudProject(userCred, self, ownerId, image, managerId)
 	return err
 }
 
-func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) (*SCachedimage, error) {
+func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) (*SCachedimage, error) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -491,12 +513,21 @@ func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userC
 	cachedImage.LastSync = time.Now().UTC()
 	cachedImage.ImageType = image.GetImageType()
 	cachedImage.ExternalId = image.GetGlobalId()
+	cachedImage.Status = image.GetStatus()
+	cachedImage.PublicScope = string(image.GetPublicScope())
+	switch image.GetPublicScope() {
+	case rbacutils.ScopeNone:
+	default:
+		cachedImage.IsPublic = true
+	}
 
 	err = manager.TableSpec().Insert(ctx, &cachedImage)
 	if err != nil {
 		return nil, err
 	}
 
+	SyncCloudProject(userCred, &cachedImage, ownerId, image, managerId)
+
 	return &cachedImage, nil
 }
 
@@ -529,7 +560,7 @@ func (image *SCachedimage) requestRefreshExternalImage(ctx context.Context, user
 		log.Errorf("iCache.GetIImageById fail %s", err)
 		return nil, err
 	}
-	err = image.syncWithCloudImage(ctx, userCred, iImage)
+	err = image.syncWithCloudImage(ctx, userCred, nil, iImage, "")
 	if err != nil {
 		log.Errorf("image.syncWithCloudImage fail %s", err)
 		return nil, err
@@ -549,7 +580,7 @@ func (image *SCachedimage) getValidStoragecache() []SStoragecache {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 
 	caches := make([]SStoragecache, 0)
 	err := db.FetchModelObjects(StoragecacheManager, q, &caches)
@@ -589,7 +620,7 @@ func (image *SCachedimage) GetUsableZoneIds() ([]string, error) {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 	q = q.Filter(sqlchemy.Equals(q.Field("status"), api.ZONE_ENABLE))
 
 	result := []string{}
@@ -632,6 +663,10 @@ func (manager *SCachedimageManager) ListItemFilter(
 	query api.CachedimageListInput,
 ) (*sqlchemy.SQuery, error) {
 	var err error
+	q, err = manager.SSharableBaseResourceManager.ListItemFilter(ctx, q, userCred, query.SharableResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
+	}
 
 	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
 		cachedImages := CachedimageManager.Query().SubQuery()
@@ -647,9 +682,9 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
 	}
 
-	q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
 	}
 
 	q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput)
@@ -667,7 +702,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -682,7 +717,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -704,9 +739,9 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.OrderByExtraFields")
 	}
 
 	return q, nil
@@ -715,10 +750,44 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 func (manager *SCachedimageManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
+	q, err = manager.SSharableVirtualResourceBaseManager.QueryDistinctExtraField(q, field)
 	if err == nil {
 		return q, nil
 	}
 
 	return q, httperrors.ErrNotFound
 }
+
+func (manager *SCachedimageManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
+	q, err := manager.SSharableVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableVirtualResourceBaseManager.L.ListItemExportKeys")
+	}
+	return q, nil
+}
+
+func (manager *SCachedimageManager) InitializeData() error {
+	images := []SCachedimage{}
+	q := manager.Query().IsNullOrEmpty("tenant_id")
+	err := db.FetchModelObjects(manager, q, &images)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range images {
+		_, err := db.Update(&images[i], func() error {
+			images[i].IsPublic = true
+			images[i].PublicScope = string(rbacutils.ScopeSystem)
+			images[i].ProjectId = "system"
+			if len(images[i].ExternalId) > 0 {
+				images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE
+			} else {
+				images[i].Status = images[i].GetStatus()
+			}
+			return nil
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update(%s)", images[i].Id)
+		}
+	}
+	return nil
+}

pkg/compute/models/cloudaccounts.go

@@ -48,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -417,9 +418,11 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 	if err != nil {
 		return output, errors.Wrap(err, "provider.GetOnPremiseIRegion")
 	}
-	hosts, err := iregion.GetIHosts()
+	// hack
+	client := iregion.(*esxi.SESXiClient)
+	hostIps, simpleVms, err := client.HostVmIPs(ctx)
 	if err != nil {
-		return output, errors.Wrap(err, "unable to get hosts")
+		return output, errors.Wrap(err, "unable to fetch ips of hosts and vms")
 	}
 	// fetch networks
 	networks := make([][]SNetwork, len(wires))
@@ -430,20 +433,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		networks[i] = nets
 	}
-	// fetch host ips
+
 	// key of ipHosts is host's ip
-	ipHosts := make(map[netutils.IPV4Addr]cloudprovider.ICloudHost, len(hosts))
-	for i := range hosts {
-		ip := hosts[i].GetAccessIp()
-		if len(ip) == 0 {
-			log.Errorf("unable to get accessip of host %q", hosts[i].GetName())
-			continue
-		}
+	ipHosts := make(map[netutils.IPV4Addr]string, len(hostIps))
+	for name, ip := range hostIps {
 		addr, err := netutils.NewIPV4Addr(ip)
 		if err != nil {
 			return output, err
 		}
-		ipHosts[addr] = hosts[i]
+		ipHosts[addr] = name
 	}
 	// Find suitable wire and the network containing the Host IP in suitable wire.
 	var (
@@ -490,15 +488,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Give the suggested network configuration for the Host IP that does not have a corresponding suitable network.
 	noNetHostIP := make([]netutils.IPV4Addr, 0, len(ipHosts))
-	for hip, host := range ipHosts {
+	for ip, name := range ipHosts {
 		rnet := api.CAHostNet{
-			Name: host.GetName(),
-			IP:   host.GetAccessIp(),
+			Name: name,
+			IP:   ip.String(),
 		}
-		if net, ok := suitableNetworks[hip]; ok {
+		if net, ok := suitableNetworks[ip]; ok {
 			rnet.SuitableNetwork = net.GetId()
 		} else {
-			noNetHostIP = append(noNetHostIP, hip)
+			noNetHostIP = append(noNetHostIP, ip)
 		}
 		output.Hosts = append(output.Hosts, rnet)
 	}
@@ -515,7 +513,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Find the suitable network containing the VM IP in Project 'input.Project', and if not, give the corresponding suggested network configuration in this project.
 	project := input.Project
-	nets := []*SNetwork{}
+	excludeNets := []*SNetwork{}
 	var allNets []SNetwork
 	if suitableWire != nil {
 		allNets, err = suitableWire.getNetworks(userCred, rbacutils.ScopeSystem)
@@ -524,85 +522,161 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		for i := range allNets {
 			if allNets[i].ProjectId == project {
-				nets = append(nets, &allNets[i])
+				excludeNets = append(excludeNets, &allNets[i])
 			}
 		}
 	}
-	// find suitable network
-	vms := make([]cloudprovider.ICloudVM, 0, len(hosts))
-	for _, host := range hosts {
-		vs, err := host.GetIVMs()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable to get VMs of host %q", host.GetName())
-		}
-		vms = append(vms, vs...)
+
+	type vm struct {
+		FakeID int
+		IP     netutils.IPV4Addr
+		Name   string
 	}
-	ipVMs := make(map[netutils.IPV4Addr]int, len(vms))
-	for i, vm := range vms {
-		nics, err := vm.GetINics()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable get nics of vm %q", vm.GetName())
-		}
-		for _, nic := range nics {
-			ipStr := nic.GetIP()
-			if len(ipStr) == 0 {
-				continue
-			}
-			ip, err := netutils.NewIPV4Addr(ipStr)
-			if err != nil {
-				return output, errors.Wrapf(err, "unable to new IPV4Addr for ip %q", nic.GetIP())
-			}
-			ipVMs[ip] = i
-		}
-		output.Guests = append(output.Guests, api.CAGuestNet{
-			Name:   vm.GetName(),
-			IPNets: make([]api.CAIPNet, 0, 1),
-		})
-	}
-	suitableNetworks = make(map[netutils.IPV4Addr]*SNetwork)
-	if len(nets) != 0 {
-		for vip := range ipVMs {
-			for i := range nets {
-				ipRange := nets[i].GetIPRange()
-				if !ipRange.Contains(vip) {
-					continue
+	vms := make([]vm, 0, len(simpleVms))
+	var nip netutils.IPV4Addr
+	guestMap := make(map[int]*api.CAGuestNet)
+	for i := range simpleVms {
+		id := i
+		if len(simpleVms[i].IPs) == 0 {
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   simpleVms[i].Name,
+					IPNets: []api.CAIPNet{},
 				}
-				suitableNetworks[vip] = nets[i]
-				break
 			}
 		}
-	}
-
-	noNetVMIPs := make([]netutils.IPV4Addr, 0, len(ipVMs))
-	for vip, i := range ipVMs {
-		ipnet := api.CAIPNet{
-			IP: vip.String(),
+		for _, ip := range simpleVms[i].IPs {
+			nip, err = netutils.NewIPV4Addr(ip)
+			if err != nil {
+				return output, err
+			}
+			vms = append(vms, vm{
+				FakeID: i,
+				IP:     nip,
+				Name:   simpleVms[i].Name,
+			})
 		}
-		if net, ok := suitableNetworks[vip]; ok {
-			ipnet.SuitableNetwork = net.GetId()
-		} else {
-			noNetVMIPs = append(noNetVMIPs, vip)
-		}
-		output.Guests[i].IPNets = append(output.Guests[ipVMs[vip]].IPNets, ipnet)
 	}
+	// sort vms via vm's ip
+	sort.Slice(vms, func(i, j int) bool {
+		return vms[i].IP < vms[j].IP
+	})
 
-	// excludedIRs is to prevent network conflicts
-	excludedIRs := make([]netutils.IPV4AddrRange, len(allNets))
-	for i := range excludedIRs {
-		excludedIRs[i] = allNets[i].GetIPRange()
+	excludeNets = make([]*SNetwork, 0, len(allNets)+len(output.HostSuggestedNetworks))
+	for i := range allNets {
+		excludeNets = append(excludeNets, &allNets[i])
 	}
 	for i := range output.HostSuggestedNetworks {
-		startIPStr := output.HostSuggestedNetworks[i].GuestIpStart
-		endIPStr := output.HostSuggestedNetworks[i].GuestIpEnd
-		startIP, _ := netutils.NewIPV4Addr(startIPStr)
-		endIP, _ := netutils.NewIPV4Addr(endIPStr)
-		excludedIRs = append(excludedIRs, netutils.NewIPV4AddrRange(startIP, endIP))
+		startipStr := output.HostSuggestedNetworks[i].GuestIpStart
+		endipStr := output.HostSuggestedNetworks[i].GuestIpEnd
+		excludeNets = append(excludeNets, &SNetwork{GuestIpEnd: endipStr, GuestIpStart: startipStr})
 	}
-	if len(noNetVMIPs) > 0 {
-		sConfs := scm.suggestVMNetwors(noNetVMIPs, excludedIRs)
-		confs := make([]api.CANetConf, len(sConfs))
+	// sort excludeNets via their GuestIpStart
+	sort.Slice(excludeNets, func(i, j int) bool {
+		return excludeNets[i].GuestIpStart < excludeNets[j].GuestIpStart
+	})
+
+	svNets := make([]netutils.IPV4AddrRange, 0, 5)
+	var vmi, neti int
+
+	lastEndIp := netutils.IPV4Addr(0)
+Loop:
+	for neti = 0; neti < len(excludeNets); {
+		startIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpStart)
+		endIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpEnd)
+		switch {
+		case vms[vmi].IP > endIp:
+			lastEndIp = endIp
+			neti++
+		case vms[vmi].IP >= startIp:
+			for vms[vmi].IP <= endIp {
+				id := vms[vmi].FakeID
+				if _, ok := guestMap[id]; !ok {
+					guestMap[id] = &api.CAGuestNet{
+						Name:   vms[vmi].Name,
+						IPNets: []api.CAIPNet{},
+					}
+				}
+				guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+					IP:              vms[vmi].IP.String(),
+					SuitableNetwork: excludeNets[neti].Id,
+				})
+				vmi += 1
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+			lastEndIp = endIp
+			neti++
+		default:
+			for vms[vmi].IP < startIp {
+				suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+				if suggestStartIp <= lastEndIp {
+					suggestStartIp = lastEndIp + 1
+				}
+				if suggestStartIp >= startIp {
+					break
+				}
+				suggestEndIp := suggestStartIp.NetAddr(24) + 255
+				if suggestEndIp >= startIp {
+					suggestEndIp = startIp - 1
+				}
+				svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+				for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+					id := vms[vmi].FakeID
+					if _, ok := guestMap[id]; !ok {
+						guestMap[id] = &api.CAGuestNet{
+							Name:   vms[vmi].Name,
+							IPNets: []api.CAIPNet{},
+						}
+					}
+					guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+						IP: vms[vmi].IP.String(),
+					})
+					vmi++
+				}
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+		}
+	}
+
+	for vmi < len(vms) {
+		suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+		if suggestStartIp <= lastEndIp {
+			suggestStartIp = lastEndIp + 1
+		}
+		suggestEndIp := suggestStartIp.NetAddr(24) + 255
+		svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+		for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+			id := vms[vmi].FakeID
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   vms[vmi].Name,
+					IPNets: []api.CAIPNet{},
+				}
+			}
+			guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+				IP: vms[vmi].IP.String(),
+			})
+			vmi++
+		}
+	}
+
+	for _, guest := range guestMap {
+		output.Guests = append(output.Guests, *guest)
+	}
+
+	if len(svNets) > 0 {
+		confs := make([]api.CANetConf, len(svNets))
 		for i := range confs {
-			confs[i].CASimpleNetConf = sConfs[i]
+			confs[i].CASimpleNetConf = api.CASimpleNetConf{
+				GuestIpStart: svNets[i].StartIp().String(),
+				GuestIpEnd:   svNets[i].EndIp().String(),
+				GuestIpMask:  24,
+				GuestGateway: (svNets[i].StartIp().NetAddr(24) + 1).String(),
+			}
 			confs[i].Name = fmt.Sprintf("%s-guest-network-%d", input.Name, i+1)
 		}
 		output.GuestSuggestedNetworks = confs

pkg/compute/models/cloudproviders.go

@@ -1394,8 +1394,7 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	params := jsonutils.NewDict()
 	task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
-		log.Errorf("%s", err)
-		return err
+		return errors.Wrapf(err, "NewTask")
 	}
 	self.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartCloudproviderDeleteTask")
 	task.ScheduleRun(nil)

pkg/compute/models/elasticips.go

@@ -426,8 +426,11 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m
 			case api.EIP_ASSOCIATE_TYPE_SERVER:
 				sq := HostManager.Query().SubQuery()
 				return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("host_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), self.ManagerId))
-			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY, api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
+			case api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
 				return q.Equals("manager_id", self.ManagerId)
+			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY:
+				sq := VpcManager.Query("id").Equals("manager_id", self.ManagerId)
+				return q.In("vpc_id", sq.SubQuery())
 			}
 			return q
 		})
@@ -1266,6 +1269,7 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		wireq := WireManager.Query().SubQuery()
 		hostwireq := HostwireManager.Query().SubQuery()
 		q := NetworkManager.Query()
+		q = NetworkManager.FilterByOwner(q, userCred, NetworkManager.NamespaceScope())
 		q = q.Join(wireq, sqlchemy.Equals(wireq.Field("id"), q.Field("wire_id")))
 		q = q.Join(hostwireq, sqlchemy.Equals(hostwireq.Field("wire_id"), wireq.Field("id")))
 		q = q.Join(hostq, sqlchemy.Equals(hostq.Field("id"), host.Id))

pkg/compute/models/guest_actions.go

@@ -3635,9 +3635,11 @@ func (self *SGuest) SaveRenewInfo(
 	guestdisks := self.GetDisks()
 	for i := 0; i < len(guestdisks); i += 1 {
 		disk := guestdisks[i].GetDisk()
-		err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
-		if err != nil {
-			return err
+		if disk.AutoDelete {
+			err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil

pkg/compute/models/guestdrivers.go

@@ -142,6 +142,8 @@ type IGuestDriver interface {
 	IsRebuildRootSupportChangeImage() bool
 	IsRebuildRootSupportChangeUEFI() bool
 
+	IsSupportdDcryptPasswordFromSecretKey() bool
+
 	RequestDeleteDetachedDisk(ctx context.Context, disk *SDisk, task taskman.ITask, isPurge bool) error
 	StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error
 	StartGuestAttachDiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error

pkg/compute/models/guests.go

@@ -4234,6 +4234,14 @@ func (self *SGuest) GetShortDesc(ctx context.Context) *jsonutils.JSONDict {
 		billingInfo.SCloudProviderInfo = host.getCloudProviderInfo()
 	}
 
+	if len(self.BackupHostId) > 0 {
+		backupHost := HostManager.FetchHostById(self.BackupHostId)
+		if backupHost != nil {
+			desc.Set("backup_host", jsonutils.NewString(backupHost.Name))
+			desc.Set("backup_host_id", jsonutils.NewString(backupHost.Id))
+		}
+	}
+
 	if priceKey := self.GetMetadata("ext:price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}
@@ -4257,40 +4265,49 @@ func (self *SGuest) saveOsType(userCred mcclient.TokenCredential, osType string)
 	return err
 }
 
+type sDeployInfo struct {
+	Os       string
+	Account  string
+	Key      string
+	Distro   string
+	Version  string
+	Arch     string
+	Language string
+}
+
 func (self *SGuest) SaveDeployInfo(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) {
+	deployInfo := sDeployInfo{}
+	data.Unmarshal(&deployInfo)
 	info := make(map[string]interface{})
-	if data.Contains("os") {
-		osName, _ := data.GetString("os")
-		self.saveOsType(userCred, osName)
-		info["os_name"] = osName
+	if len(deployInfo.Os) > 0 {
+		self.saveOsType(userCred, deployInfo.Os)
+		info["os_name"] = deployInfo.Os
 	}
-	if data.Contains("account") {
-		account, _ := data.GetString("account")
-		info["login_account"] = account
-		if data.Contains("key") {
-			key, _ := data.GetString("key")
-			info["login_key"] = key
+	driver := self.GetDriver()
+	if len(deployInfo.Account) > 0 {
+		info["login_account"] = deployInfo.Account
+		if len(deployInfo.Key) > 0 {
+			info["login_key"] = deployInfo.Key
+			if len(self.KeypairId) > 0 && !driver.IsSupportdDcryptPasswordFromSecretKey() { // Tencent Cloud does not support simultaneous setting of secret keys and passwords
+				info["login_key"], _ = seclib2.EncryptBase64(self.GetKeypairPublicKey(), "")
+			}
 			info["login_key_timestamp"] = timeutils.UtcNow()
 		} else {
 			info["login_key"] = "none"
 			info["login_key_timestamp"] = "none"
 		}
 	}
-	if data.Contains("distro") {
-		dist, _ := data.GetString("distro")
-		info["os_distribution"] = dist
+	if len(deployInfo.Distro) > 0 {
+		info["os_distribution"] = deployInfo.Distro
 	}
-	if data.Contains("version") {
-		ver, _ := data.GetString("version")
-		info["os_version"] = ver
+	if len(deployInfo.Version) > 0 {
+		info["os_version"] = deployInfo.Version
 	}
-	if data.Contains("arch") {
-		arch, _ := data.GetString("arch")
-		info["os_arch"] = arch
+	if len(deployInfo.Arch) > 0 {
+		info["os_arch"] = deployInfo.Arch
 	}
-	if data.Contains("language") {
-		lang, _ := data.GetString("language")
-		info["os_language"] = lang
+	if len(deployInfo.Language) > 0 {
+		info["os_language"] = deployInfo.Language
 	}
 	self.SetAllMetadata(ctx, info, userCred)
 	self.saveOldPassword(ctx, userCred)

pkg/compute/models/hosts.go

@@ -5450,3 +5450,15 @@ func (manager *SHostManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SHostManager) FetchHostByExtId(extid string) *SHost {
+	host := SHost{}
+	host.SetModelManager(manager, &host)
+	err := manager.Query().Equals("external_id", extid).First(&host)
+	if err != nil {
+		log.Errorf("fetchHostByExtId fail %s", err)
+		return nil
+	} else {
+		return &host
+	}
+}

pkg/compute/models/initdb.go

@@ -60,6 +60,8 @@ func InitDB() error {
 
 		ScheduledTaskActivityManager,
 		ExternalProjectManager,
+		CachedimageManager,
+		StoragecachedimageManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/secgroupcache.go

@@ -355,6 +355,39 @@ func (self *SSecurityGroupCache) GetSecgroup() (*SSecurityGroup, error) {
 	return model.(*SSecurityGroup), nil
 }
 
+func (self *SSecurityGroupCache) syncWithCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudSecurityGroup) error {
+	_, err := db.Update(self, func() error {
+		self.Status = api.SECGROUP_CACHE_STATUS_READY
+		self.Name = ext.GetName()
+		self.Description = ext.GetDescription()
+		self.ExternalProjectId = ext.GetProjectId()
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "db.Update")
+	}
+	secgroup, err := self.GetSecgroup()
+	if err != nil {
+		return errors.Wrapf(err, "GetSecurity")
+	}
+	cacheCount, err := secgroup.GetSecgroupCacheCount()
+	if err != nil {
+		return errors.Wrapf(err, "GetSecgroupCacheCount")
+	}
+	if cacheCount > 1 {
+		return nil
+	}
+	info, err := SecurityGroupManager.getRuleInfo(provider, ext)
+	if err != nil {
+		return errors.Wrapf(err, "getRuleInfo")
+	}
+	err = secgroup.SyncSecurityGroupRules(ctx, userCred, info)
+	if err != nil {
+		return errors.Wrapf(err, "SyncSecurityGroupRules")
+	}
+	return nil
+}
+
 func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
@@ -415,25 +448,18 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 	}
 
 	for i := 0; i < len(commondb); i++ {
-		_, err = db.Update(&commondb[i], func() error {
-			commondb[i].Status = api.SECGROUP_CACHE_STATUS_READY
-			commondb[i].Name = commonext[i].GetName()
-			commondb[i].Description = commonext[i].GetDescription()
-			commondb[i].ExternalProjectId = commonext[i].GetProjectId()
-			return nil
-		})
+		err = commondb[i].syncWithCloudSecurityGroup(ctx, userCred, provider, commonext[i])
 		if err != nil {
-			syncResult.UpdateError(err)
-		} else {
-			syncResult.Update()
+			syncResult.UpdateError(errors.Wrapf(err, "syncWithCloudSecurityGroup"))
+			continue
 		}
+		syncResult.Update()
 	}
 
-	//相同的不能同步, 原因: 多个平台的安全组可能共用一个本地安全组,下面仅仅是新加的安全组
 	for i := 0; i < len(added); i++ {
 		secgroup, err := SecurityGroupManager.newFromCloudSecgroup(ctx, userCred, provider, added[i])
 		if err != nil {
-			syncResult.AddError(err)
+			syncResult.AddError(errors.Wrapf(err, "newFromCloudSecgroup"))
 			continue
 		}
 		if secgroup.ProjectId != provider.ProjectId {
@@ -450,7 +476,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 		}
 		cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id, added[i].GetProjectId())
 		if err != nil {
-			syncResult.AddError(fmt.Errorf("failed to create secgroup cache for secgroup %s(%s) provider: %s: %s", secgroup.Name, secgroup.Name, provider.Name, err))
+			syncResult.AddError(errors.Wrapf(err, "NewCache for secgroup %s provider %s", secgroup.Name, provider.Name))
 			continue
 		}
 		_, err = db.Update(cache, func() error {
@@ -461,7 +487,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 			return nil
 		})
 		if err != nil {
-			syncResult.AddError(err)
+			syncResult.AddError(errors.Wrapf(err, "db.Update"))
 			continue
 		}
 		localSecgroups = append(localSecgroups, *secgroup)
@@ -560,3 +586,20 @@ func (manager *SSecurityGroupCacheManager) ListItemExportKeys(ctx context.Contex
 
 	return q, nil
 }
+
+func (self *SSecurityGroupCache) GetISecurityGroup() (cloudprovider.ICloudSecurityGroup, error) {
+	if len(self.ExternalId) == 0 {
+		return nil, errors.Wrapf(cloudprovider.ErrNotFound, "empty externalId")
+	}
+
+	manager := self.GetCloudprovider()
+	if manager == nil {
+		return nil, errors.Wrapf(cloudprovider.ErrNotFound, "failed to found manager")
+	}
+
+	iRegion, err := self.GetIRegion()
+	if err != nil {
+		return nil, errors.Wrapf(err, "GetIRegion")
+	}
+	return iRegion.GetISecurityGroupById(self.ExternalId)
+}

pkg/compute/models/secgrouprules.go

@@ -22,7 +22,6 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/util/stringutils"
@@ -462,8 +461,7 @@ func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSec
 	return rules, nil
 }
 
-func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules cloudprovider.SecurityRuleSet) compare.SyncResult {
-	syncResult := compare.SyncResult{}
+func (self *SSecurityGroup) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, rules cloudprovider.SecurityRuleSet) error {
 	priority, prePriority := 10, 0
 	for i := 0; i < len(rules); i++ {
 		// 这里避免了Rule规则优先级在 1-100之外的问题,ext.GetRules()不需要进行优先级转换
@@ -472,19 +470,17 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre
 		}
 		prePriority = rules[i].Priority
 		rules[i].Priority = priority
-		_, err := manager.newFromCloudSecurityGroup(ctx, userCred, rules[i], secgroup)
+		_, err := self.newFromCloudSecurityGroupRule(ctx, userCred, rules[i])
 		if err != nil {
-			syncResult.AddError(err)
-			continue
+			return errors.Wrapf(err, "newFromCloudSecurityGroupRule")
 		}
-		syncResult.Add()
 	}
-	return syncResult
+	return nil
 }
 
-func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) {
-	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
-	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
+func (self *SSecurityGroup) newFromCloudSecurityGroupRule(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule) (*SSecurityGroupRule, error) {
+	lockman.LockObject(ctx, self)
+	defer lockman.ReleaseObject(ctx, self)
 
 	protocol := rule.Protocol
 	if len(protocol) == 0 {
@@ -505,11 +501,11 @@ func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.
 		Action:      string(rule.Action),
 		Description: rule.Description,
 	}
-	secrule.SecgroupId = secgroup.Id
+	secrule.SecgroupId = self.Id
 
-	err := manager.TableSpec().Insert(ctx, secrule)
+	err := SecurityGroupRuleManager.TableSpec().Insert(ctx, secrule)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "SecurityGroupRuleManager.Insert")
 	}
 	return secrule, nil
 }

pkg/compute/models/secgroups.go

@@ -37,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
@@ -539,6 +540,14 @@ func totalSecurityGroupCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdent
 	return q.CountWithError()
 }
 
+func (self *SSecurityGroup) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "purge")
+}
+
+func (self *SSecurityGroup) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return nil, self.StartDeleteSecurityGroupTask(ctx, userCred, true, "")
+}
+
 func (self *SSecurityGroup) AllowPerformUncacheSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "uncache-secgroup")
 }
@@ -821,7 +830,57 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err
 	}
 }
 
-func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) {
+func (self *SSecurityGroup) cleanRules(ctx context.Context, userCred mcclient.TokenCredential) error {
+	rules := []SSecurityGroupRule{}
+	q := SecurityGroupRuleManager.Query().Equals("secgroup_id", self.Id)
+	err := db.FetchModelObjects(SecurityGroupRuleManager, q, &rules)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range rules {
+		err = rules[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "DeleteRule(%s)", rules[i].Id)
+		}
+	}
+	return nil
+}
+
+func (self *SSecurityGroup) SyncSecurityGroupRules(ctx context.Context, userCred mcclient.TokenCredential, info *sRuleInfo) error {
+	inRules := cloudprovider.AddDefaultRule(info.inRules, info.defaultInRule, "in:deny any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules)
+	cloudprovider.SortSecurityRule(inRules, info.order, info.onlyAllowRules)
+	outRules := cloudprovider.AddDefaultRule(info.outRules, info.defaultOutRule, "out:allow any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules)
+	cloudprovider.SortSecurityRule(outRules, info.order, info.onlyAllowRules)
+
+	err := self.cleanRules(ctx, userCred)
+	if err != nil {
+		return errors.Wrapf(err, "cleanRules")
+	}
+
+	err = self.SyncRules(ctx, userCred, inRules)
+	if err != nil {
+		return errors.Wrapf(err, "SyncInRules")
+	}
+	err = self.SyncRules(ctx, userCred, outRules)
+	if err != nil {
+		return errors.Wrapf(err, "SyncOutRules")
+	}
+	return nil
+}
+
+type sRuleInfo struct {
+	rules          []cloudprovider.SecurityRule
+	inRules        []cloudprovider.SecurityRule
+	outRules       []cloudprovider.SecurityRule
+	defaultInRule  cloudprovider.SecurityRule
+	defaultOutRule cloudprovider.SecurityRule
+	order          cloudprovider.TPriorityOrder
+	onlyAllowRules bool
+	maxPriority    int
+	minPriority    int
+}
+
+func (manager *SSecurityGroupManager) getRuleInfo(provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*sRuleInfo, error) {
 	regionDriver, err := provider.GetRegionDriver()
 	if err != nil {
 		return nil, errors.Wrap(err, "provider.GetRegionDriver")
@@ -832,36 +891,48 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, errors.Wrap(err, "extSec.GetRules")
 	}
 
-	inRules := []cloudprovider.SecurityRule{}
-	outRules := []cloudprovider.SecurityRule{}
+	info := &sRuleInfo{
+		rules:          rules,
+		inRules:        []cloudprovider.SecurityRule{},
+		outRules:       []cloudprovider.SecurityRule{},
+		defaultInRule:  regionDriver.GetDefaultSecurityGroupInRule(),
+		defaultOutRule: regionDriver.GetDefaultSecurityGroupOutRule(),
+		order:          regionDriver.GetSecurityGroupRuleOrder(),
+		onlyAllowRules: regionDriver.IsOnlySupportAllowRules(),
+		maxPriority:    regionDriver.GetSecurityGroupRuleMaxPriority(),
+		minPriority:    regionDriver.GetSecurityGroupRuleMinPriority(),
+	}
+
 	for i := range rules {
 		if rules[i].Direction == secrules.DIR_IN {
-			inRules = append(inRules, rules[i])
+			info.inRules = append(info.inRules, rules[i])
 		} else {
-			outRules = append(outRules, rules[i])
+			info.outRules = append(info.outRules, rules[i])
 		}
 	}
+	return info, nil
+}
 
-	maxPriority := regionDriver.GetSecurityGroupRuleMaxPriority()
-	minPriority := regionDriver.GetSecurityGroupRuleMinPriority()
-
-	defaultInRule := regionDriver.GetDefaultSecurityGroupInRule()
-	defaultOutRule := regionDriver.GetDefaultSecurityGroupOutRule()
-	order := regionDriver.GetSecurityGroupRuleOrder()
-	onlyAllowRules := regionDriver.IsOnlySupportAllowRules()
-
-	// 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组
-	secgroups := []SSecurityGroup{}
-	q := manager.Query().Equals("domain_id", provider.DomainId)
-	err = db.FetchModelObjects(manager, q, &secgroups)
+func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) {
+	info, err := manager.getRuleInfo(provider, extSec)
 	if err != nil {
-		return nil, errors.Wrap(err, "db.FetchModelObjects")
+		return nil, errors.Wrapf(err, "getRuleInfo")
 	}
-	for i := range secgroups {
-		localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules(""))
-		_, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
-		if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 {
-			return &secgroups[i], nil
+
+	if options.Options.EnableAutoMergeSecurityGroup {
+		// 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组
+		secgroups := []SSecurityGroup{}
+		q := manager.Query().Equals("domain_id", provider.DomainId)
+		err = db.FetchModelObjects(manager, q, &secgroups)
+		if err != nil {
+			return nil, errors.Wrap(err, "db.FetchModelObjects")
+		}
+		for i := range secgroups {
+			localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules(""))
+			_, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(info.minPriority, info.maxPriority, info.order, localRules, info.rules, info.defaultInRule, info.defaultOutRule, info.onlyAllowRules, false)
+			if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 {
+				return &secgroups[i], nil
+			}
 		}
 	}
 
@@ -870,31 +941,26 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 
 	secgroup := SSecurityGroup{}
 	secgroup.SetModelManager(manager, &secgroup)
-	newName, err := db.GenerateName(manager, userCred, extSec.GetName())
+	secgroup.Name, err = db.GenerateName(manager, userCred, extSec.GetName())
 	if err != nil {
 		return nil, err
 	}
 
-	secgroup.Name = newName
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId
 
-	if err := manager.TableSpec().Insert(ctx, &secgroup); err != nil {
-		return nil, err
+	err = manager.TableSpec().Insert(ctx, &secgroup)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Insert")
 	}
 
-	//这里必须先同步下规则,不然下次对比此安全组规则为空
-	inRules = cloudprovider.AddDefaultRule(inRules, defaultInRule, "in:deny any", order, minPriority, maxPriority, onlyAllowRules)
-	cloudprovider.SortSecurityRule(inRules, order, onlyAllowRules)
-	outRules = cloudprovider.AddDefaultRule(outRules, defaultOutRule, "out:allow any", order, minPriority, maxPriority, onlyAllowRules)
-	cloudprovider.SortSecurityRule(outRules, order, onlyAllowRules)
-
-	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, inRules)
-	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, outRules)
+	err = secgroup.SyncSecurityGroupRules(ctx, userCred, info)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SyncSecurityGroupRules")
+	}
 
 	db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred)
-
 	return &secgroup, nil
 }
 
@@ -1035,10 +1101,12 @@ func (self *SSecurityGroup) GetSecurityGroupCaches() []SSecurityGroupCache {
 }
 
 func (self *SSecurityGroup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	return self.StartDeleteSecurityGroupTask(ctx, userCred, jsonutils.NewDict(), "")
+	return self.StartDeleteSecurityGroupTask(ctx, userCred, false, "")
 }
 
-func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error {
+func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, isPurge bool, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewBool(isPurge), "purge")
 	self.SetStatus(userCred, api.SECGROUP_STATUS_DELETING, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "SecurityGroupDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {

pkg/compute/models/storagecachedimages.go

@@ -331,7 +331,7 @@ func (self *SStoragecachedimage) markDeleting(ctx context.Context, userCred mccl
 	}
 
 	if !isForce && !utils.IsInStringArray(self.Status,
-		[]string{api.CACHED_IMAGE_STATUS_READY, api.CACHED_IMAGE_STATUS_DELETING, api.CACHED_IMAGE_STATUS_CACHE_FAILED}) {
+		[]string{api.CACHED_IMAGE_STATUS_ACTIVE, api.CACHED_IMAGE_STATUS_DELETING, api.CACHED_IMAGE_STATUS_CACHE_FAILED}) {
 		return httperrors.NewInvalidStatusError("Cannot uncache in status %s", self.Status)
 	}
 	_, err = db.Update(self, func() error {
@@ -439,17 +439,17 @@ func (self *SStoragecachedimage) syncRemoveCloudImage(ctx context.Context, userC
 	return nil
 }
 
-func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error {
+func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	cachedImage := self.GetCachedimage()
 	if len(cachedImage.ExternalId) > 0 {
 		self.SetStatus(userCred, image.GetStatus(), "")
-		return cachedImage.syncWithCloudImage(ctx, userCred, image)
+		return cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, managerId)
 	} else {
 		return nil
 	}
 }
 
-func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage, cache *SStoragecache) error {
+func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, cache *SStoragecache) error {
 	var cachedImage *SCachedimage
 	imgObj, err := db.FetchByExternalId(CachedimageManager, image.GetGlobalId())
 	if err != nil {
@@ -472,7 +472,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context
 		}
 		if cachedImage == nil {
 			// no such image
-			cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, image)
+			cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, ownerId, image, cache.ManagerId)
 			if err != nil {
 				log.Errorf("CachedimageManager.newFromCloudImage fail %s", err)
 				return err
@@ -482,7 +482,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context
 		cachedImage = imgObj.(*SCachedimage)
 	}
 	if len(cachedImage.ExternalId) > 0 {
-		cachedImage.syncWithCloudImage(ctx, userCred, image)
+		cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, cache.ManagerId)
 	}
 	scimg := manager.Register(ctx, userCred, cache.GetId(), cachedImage.GetId(), image.GetStatus())
 	if scimg == nil {
@@ -553,3 +553,22 @@ func (manager *SStoragecachedimageManager) OrderByExtraFields(
 
 	return q, nil
 }
+
+func (manager *SStoragecachedimageManager) InitializeData() error {
+	images := []SStoragecachedimage{}
+	q := manager.Query().Equals("status", "ready")
+	err := db.FetchModelObjects(manager, q, &images)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range images {
+		_, err := db.Update(&images[i], func() error {
+			images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE
+			return nil
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update(%d)", images[i].RowId)
+		}
+	}
+	return nil
+}

pkg/compute/models/storagecaches.go

@@ -610,6 +610,13 @@ func (cache *SStoragecache) SyncCloudImages(
 		return syncResult
 	}
 
+	var syncOwnerId mcclient.IIdentityProvider
+
+	provider := cache.GetCloudprovider()
+	if provider != nil {
+		syncOwnerId = provider.GetOwnerId()
+	}
+
 	removed := make([]SStoragecachedimage, 0)
 	commondb := make([]SStoragecachedimage, 0)
 	commonext := make([]cloudprovider.ICloudImage, 0)
@@ -631,7 +638,7 @@ func (cache *SStoragecache) SyncCloudImages(
 		}
 	}
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudImage(ctx, userCred, commonext[i])
+		err = commondb[i].syncWithCloudImage(ctx, userCred, syncOwnerId, commonext[i], cache.ManagerId)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -639,7 +646,7 @@ func (cache *SStoragecache) SyncCloudImages(
 		}
 	}
 	for i := 0; i < len(added); i += 1 {
-		err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, added[i], cache)
+		err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, syncOwnerId, added[i], cache)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
@@ -662,7 +669,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool {
 		log.Debugf("image %s is not a customized image, no need to cache", imageId)
 		return false
 	}
-	cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY})
+	cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE})
 	for i := range cachedImages {
 		if cachedImages[i].Id == imageId {
 			// already cached
@@ -675,7 +682,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool {
 }
 
 func (self *SStoragecache) StartRelinquishLeastUsedCachedImageTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, parentTaskId string) error {
-	cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY})
+	cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE})
 	leastUsedIdx := -1
 	leastRefCount := -1
 	for i := range cachedImages {

pkg/compute/models/wires.go

@@ -454,6 +454,7 @@ func (manager *SWireManager) totalCountQ(
 ) *sqlchemy.SQuery {
 	guestsQ := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId)
 	guests := guestsQ.SubQuery()
+
 	hostsQ := HostManager.Query()
 	if len(hostTypes) > 0 {
 		hostsQ = hostsQ.In("host_type", hostTypes)
@@ -465,7 +466,9 @@ func (manager *SWireManager) totalCountQ(
 		hostsQ = RangeObjectsFilter(hostsQ, rangeObjs, nil, hostsQ.Field("zone_id"), hostsQ.Field("manager_id"), hostsQ.Field("id"), nil)
 	}
 	hosts := hostsQ.SubQuery()
+
 	groups := filterByScopeOwnerId(GroupManager.Query(), scope, ownerId).SubQuery()
+
 	lbsQ := filterByScopeOwnerId(LoadbalancerManager.Query(), scope, ownerId)
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		lbsQ = CloudProviderFilter(lbsQ, lbsQ.Field("manager_id"), providers, brands, cloudEnv)
@@ -610,21 +613,20 @@ func (manager *SWireManager) totalCountQ(
 	)
 	q = q.LeftJoin(netSQ, sqlchemy.Equals(wires.Field("id"), netSQ.Field("wire_id")))
 
-	if len(rangeObjs) > 0 || len(hostTypes) > 0 {
+	if len(hostTypes) > 0 {
 		hostwires := HostwireManager.Query().SubQuery()
-		sq := hostwires.Query(hostwires.Field("wire_id"))
-		sq = sq.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id")))
-		sq = sq.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-		sq = AttachUsageQuery(sq, hosts, hostTypes, nil, nil, nil, "", rangeObjs)
-		q = q.Filter(sqlchemy.In(wires.Field("id"), sq.Distinct()))
+		hosts := HostManager.Query().SubQuery()
+		q = q.Join(hostwires, sqlchemy.Equals(q.Field("id"), hostwires.Field("wire_id")))
+		q = q.Join(hosts, sqlchemy.Equals(hostwires.Field("host_id"), hosts.Field("id")))
+		q = q.Filter(sqlchemy.In(hosts.Field("host_type"), hostTypes))
 	}
 
-	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+	if len(rangeObjs) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		vpcs := VpcManager.Query().SubQuery()
 
-		subq := vpcs.Query(vpcs.Field("id"))
-		subq = CloudProviderFilter(subq, vpcs.Field("manager_id"), providers, brands, cloudEnv)
-		q = q.Filter(sqlchemy.In(wires.Field("vpc_id"), subq.SubQuery()))
+		q = q.Join(vpcs, sqlchemy.Equals(q.Field("vpc_id"), vpcs.Field("id")))
+		q = CloudProviderFilter(q, vpcs.Field("manager_id"), providers, brands, cloudEnv)
+		q = RangeObjectsFilter(q, rangeObjs, vpcs.Field("cloudregion_id"), q.Field("zone_id"), vpcs.Field("manager_id"), nil, nil)
 	}
 
 	return q

pkg/compute/options/options.go

@@ -144,6 +144,8 @@ type ComputeOptions struct {
 	SASControllerOptions
 	common_options.CommonOptions
 	common_options.DBOptions
+
+	EnableAutoMergeSecurityGroup bool `help:"Enable auto merge secgroup when sync security group from cloud, default False" default:"false"`
 }
 
 type SCapabilityOptions struct {

pkg/compute/tasks/security_group_cache_delete_task.go

@@ -16,7 +16,6 @@ package tasks
 
 import (
 	"context"
-	"database/sql"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -26,7 +25,6 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
-	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SecurityGroupCacheDeleteTask struct {
@@ -39,10 +37,6 @@ func init() {
 
 func (self *SecurityGroupCacheDeleteTask) taskFailed(ctx context.Context, cache *models.SSecurityGroupCache, err error) {
 	cache.SetStatus(self.UserCred, api.SECGROUP_CACHE_STATUS_DELETE_FAILED, err.Error())
-	secgroup, _ := cache.GetSecgroup()
-	if secgroup != nil {
-		logclient.AddActionLogWithStartable(self, secgroup, logclient.ACT_DELETE, err, self.UserCred, false)
-	}
 	self.SetStageFailed(ctx, err.Error())
 }
 
@@ -54,38 +48,18 @@ func (self *SecurityGroupCacheDeleteTask) taskComplete(ctx context.Context, cach
 func (self *SecurityGroupCacheDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	cache := obj.(*models.SSecurityGroupCache)
 
-	if len(cache.ExternalId) == 0 {
-		self.taskComplete(ctx, cache)
-		return
-	}
-
-	_, err := models.CloudproviderManager.FetchById(cache.ManagerId)
-	if err == sql.ErrNoRows {
-		self.taskComplete(ctx, cache)
-		return
-	}
-
-	iRegion, err := cache.GetIRegion()
+	iSecgroup, err := cache.GetISecurityGroup()
 	if err != nil {
 		if errors.Cause(err) == cloudprovider.ErrNotFound {
 			self.taskComplete(ctx, cache)
 			return
 		}
-		self.taskFailed(ctx, cache, errors.Wrap(err, "cache.GetIRegion"))
-		return
-	}
-	iSecgroup, err := iRegion.GetISecurityGroupById(cache.ExternalId)
-	if err != nil {
-		if err == cloudprovider.ErrNotFound {
-			self.taskComplete(ctx, cache)
-			return
-		}
-		self.taskFailed(ctx, cache, errors.Wrap(err, "iRegion.GetIStoragecacheById"))
+		self.taskFailed(ctx, cache, errors.Wrapf(err, "GetISecurityGroup"))
 		return
 	}
 	err = iSecgroup.Delete()
 	if err != nil {
-		self.taskFailed(ctx, cache, err)
+		self.taskFailed(ctx, cache, errors.Wrapf(err, "iSecgroup.Delete"))
 		return
 	}
 	self.taskComplete(ctx, cache)

pkg/compute/tasks/security_group_delete_task.go

@@ -18,11 +18,14 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SecurityGroupDeleteTask struct {
@@ -33,37 +36,38 @@ func init() {
 	taskman.RegisterTask(SecurityGroupDeleteTask{})
 }
 
-func (self *SecurityGroupDeleteTask) getErrorCount() int64 {
-	count, _ := self.GetParams().Int("faild_count")
-	return count
-}
-
-func (self *SecurityGroupDeleteTask) addErrorCount() {
-	count := self.getErrorCount()
-	count += 1
-	self.GetParams().Set("failed_count", jsonutils.NewInt(count))
+func (self *SecurityGroupDeleteTask) taskFailed(ctx context.Context, secgroup *models.SSecurityGroup, err error) {
+	secgroup.SetStatus(self.UserCred, api.SECGROUP_STATUS_READY, "")
+	logclient.AddActionLogWithContext(ctx, secgroup, logclient.ACT_DELOCATE, err, self.UserCred, false)
+	self.SetStageFailed(ctx, err.Error())
 }
 
 func (self *SecurityGroupDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
-	self.SetStage("OnSecurityGroupUncacheComplete", nil)
-	self.OnSecurityGroupUncacheComplete(ctx, obj, data)
-}
-
-func (self *SecurityGroupDeleteTask) OnSecurityGroupUncacheComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	secgroup := obj.(*models.SSecurityGroup)
-	secgroupCaches := secgroup.GetSecurityGroupCaches()
-	errCount := self.getErrorCount()
-	if len(secgroupCaches) == int(errCount) {
-		if errCount == 0 {
-			secgroup.RealDelete(ctx, self.UserCred)
-		}
-		secgroup.SetStatus(self.UserCred, api.SECGROUP_STATUS_READY, "")
-		self.SetStageComplete(ctx, nil)
-		return
-	}
-	secgroupCaches[errCount].StartSecurityGroupCacheDeleteTask(ctx, self.UserCred, self.GetTaskId())
-}
+	caches := secgroup.GetSecurityGroupCaches()
 
-func (self *SecurityGroupDeleteTask) OnSecurityGroupUncacheCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
-	self.addErrorCount()
+	isPurge := jsonutils.QueryBoolean(self.Params, "purge", false)
+
+	for i := range caches {
+		if !isPurge {
+			iSecgroup, err := caches[i].GetISecurityGroup()
+			if err != nil {
+				if errors.Cause(err) == cloudprovider.ErrNotFound {
+					caches[i].RealDelete(ctx, self.GetUserCred())
+					continue
+				}
+				self.taskFailed(ctx, secgroup, errors.Wrapf(err, "GetISecurityGroup for cache %s(%s)", caches[i].Name, caches[i].Id))
+				return
+			}
+			err = iSecgroup.Delete()
+			if err != nil {
+				self.taskFailed(ctx, secgroup, errors.Wrapf(err, "iSecgroup.Delete"))
+				return
+			}
+		}
+		caches[i].RealDelete(ctx, self.GetUserCred())
+	}
+
+	secgroup.RealDelete(ctx, self.GetUserCred())
+	self.SetStageComplete(ctx, nil)
 }

pkg/compute/tasks/storage_cache_image_task.go

@@ -57,7 +57,7 @@ func (self *StorageCacheImageTask) OnRelinquishLeastUsedCachedImageComplete(ctx
 	storageCache := obj.(*models.SStoragecache)
 
 	scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId, "")
-	if scimg.Status != api.CACHED_IMAGE_STATUS_READY {
+	if scimg.Status != api.CACHED_IMAGE_STATUS_ACTIVE {
 		scimg.SetStatus(self.UserCred, api.CACHED_IMAGE_STATUS_CACHING, "storage_cache_image_task")
 	}
 
@@ -117,7 +117,7 @@ func (self *StorageCacheImageTask) OnCacheSucc(ctx context.Context, cache *model
 	scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, cache.Id, imageId, "")
 	extImgId, _ := data.GetString("image_id")
 
-	scimg.SetStatus(self.UserCred, api.CACHED_IMAGE_STATUS_READY, "cached")
+	scimg.SetStatus(self.UserCred, api.CACHED_IMAGE_STATUS_ACTIVE, "cached")
 	if len(extImgId) > 0 && scimg.ExternalId != extImgId {
 		scimg.SetExternalId(extImgId)
 	}

pkg/esxi/agent.go

@@ -133,7 +133,6 @@ func Start(app *appsrv.Application) error {
 }
 
 func (agent *SEsxiAgent) AddImageCacheHandler(prefix string, app *appsrv.Application) {
-	hostutils.InitWorkerManager()
 	app.AddHandler("POST",
 		fmt.Sprintf("%s/disks/image_cache", prefix),
 		auth.Authenticate(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

pkg/esxi/options/options.go

@@ -19,18 +19,19 @@ import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 type EsxiOptions struct {
 	common_options.CommonOptions
 
-	ListenInterface         string `help:"Master address of host server" default:"br0"`
-	ListenAddress           string `help:"Host serve IP address to select when multiple address bind to ListenInterface"`
-	EsxiAgentPath           string `default:"/opt/cloud/workspace/esxi_agent" help:"Path for esxi agent configuration files"`
-	ImageCachePath          string `help:"Path for storing image caches"`
-	ImageCacheLimit         int    `help:"Maximal storage space for image caching, in GB" default:"20"`
-	AgentTempPath           string `help:"Path for ESXI Agent"`
-	AgentTempLimit          int    `help:"Maximal storage space for ESXi agent, in GB" default:"20"`
-	LinuxDefaultRootUser    bool   `help:"Default account for Linux system is root" default:"false"`
-	WindowsDefaultAdminUser bool   `help:"Default account for Windows system is Administrator" default:"true"`
-	DefaultImageSaveFormat  string `help:"Default image save format, default is vmdk, canbe qcow2" default:"vmdk"`
-	Zone                    string `help:"Zone where the agent locates"`
-	DeployServerSocketPath  string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	ListenInterface          string `help:"Master address of host server" default:"br0"`
+	ListenAddress            string `help:"Host serve IP address to select when multiple address bind to ListenInterface"`
+	EsxiAgentPath            string `default:"/opt/cloud/workspace/esxi_agent" help:"Path for esxi agent configuration files"`
+	ImageCachePath           string `help:"Path for storing image caches"`
+	ImageCacheLimit          int    `help:"Maximal storage space for image caching, in GB" default:"20"`
+	AgentTempPath            string `help:"Path for ESXI Agent"`
+	AgentTempLimit           int    `help:"Maximal storage space for ESXi agent, in GB" default:"20"`
+	LinuxDefaultRootUser     bool   `help:"Default account for Linux system is root" default:"false"`
+	WindowsDefaultAdminUser  bool   `help:"Default account for Windows system is Administrator" default:"true"`
+	DefaultImageSaveFormat   string `help:"Default image save format, default is vmdk, canbe qcow2" default:"vmdk"`
+	Zone                     string `help:"Zone where the agent locates"`
+	DeployServerSocketPath   string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	HostDelayTaskWorkerCount int    `default:"8" help:"Host delay worker thread count, default is 8"`
 }
 
 var (

pkg/esxi/service/esxi_agent_service.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/esxi/options"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 )
 
 type SExsiAgentService struct {
@@ -70,6 +71,7 @@ func (s *SExsiAgentService) StartService() {
 	fsdriver.Init(nil)
 	deployclient.Init(options.Options.DeployServerSocketPath)
 
+	hostutils.InitWorkerManagerWithCount(options.Options.HostDelayTaskWorkerCount)
 	app := app_common.InitApp(&options.Options.BaseOptions, false)
 	handler.InitHandlers(app)
 

pkg/hostman/guestfs/fsdriver/linux.go

@@ -773,7 +773,7 @@ func (d *SUbuntuRootFs) GetReleaseInfo(rootFs IDiskPartition) *deployapi.Release
 	lines := strings.Split(string(rel), "\n")
 	for _, l := range lines {
 		if strings.HasPrefix(l, distroKey) {
-			version = strings.TrimSpace(l[len(distroKey) : len(l)-1])
+			version = strings.TrimSpace(l[len(distroKey):])
 		}
 	}
 	return deployapi.NewReleaseInfo(d.GetName(), version, d.GetArch(rootFs))

pkg/hostman/hostutils/hostutils.go

@@ -189,7 +189,11 @@ func DelayTaskWithWorker(
 }
 
 func InitWorkerManager() {
-	wm = workmanager.NewWorkManger(TaskFailed, TaskComplete, options.HostOptions.DefaultRequestWorkerCount)
+	InitWorkerManagerWithCount(options.HostOptions.DefaultRequestWorkerCount)
+}
+
+func InitWorkerManagerWithCount(count int) {
+	wm = workmanager.NewWorkManger(TaskFailed, TaskComplete, count)
 }
 
 func InitK8sWorkerManager() {

pkg/hostman/storageman/imagecache_rbd.go

@@ -24,7 +24,6 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
-	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/remotefile"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -75,11 +74,11 @@ func (r *SRbdImageCache) Acquire(ctx context.Context, zone, srcUrl, format strin
 	}
 	r.imageName = localImageCache.GetName()
 	if !r.Load() {
-		log.Debugf("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath())
+		log.Infof("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath())
 		err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuImg(),
 			"convert", "-O", "raw", localImageCache.GetPath(), r.GetPath()).Run()
 		if err != nil {
-			log.Errorf("failed to convert image %s", options.HostOptions.ServersPath)
+			log.Errorf("failed to convert image %s", err)
 			return false
 		}
 	}

pkg/hostman/storageman/imagecachemanager_agent.go

@@ -225,12 +225,12 @@ func (c *SAgentImageCacheManager) perfetchTemplateVMImageCache(ctx context.Conte
 	if err != nil {
 		return nil, errors.Wrap(err, "host.GetDatacenter")
 	}
-	_, err = dc.GetTemplateVMById(data.ImageExternalId)
+	_, err = dc.FetchTemplateVMById(data.ImageExternalId)
 	if err != nil {
 		return nil, err
 	}
 	res := jsonutils.NewDict()
-	res.Add(jsonutils.NewString(data.ImageId), "image_id")
+	res.Add(jsonutils.NewString(data.ImageExternalId), "image_id")
 	return res, nil
 }
 

pkg/hostman/storageman/imagecachemanager_rbd.go

@@ -111,7 +111,7 @@ func (c *SRbdImageCacheManager) PrefetchImageCache(ctx context.Context, data int
 	if err != nil {
 		return nil, err
 	}
-	format := "qcow2"
+	format, _ := body.GetString("format")
 	srcUrl, _ := body.GetString("src_url")
 	zone, _ := body.GetString("zone")
 

pkg/keystone/tokens/auth.go

@@ -17,7 +17,6 @@ package tokens
 import (
 	"context"
 	"database/sql"
-	"fmt"
 	"time"
 
 	"yunion.io/x/pkg/errors"
@@ -146,8 +145,12 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 
 	idp := idpObj.(*models.SIdentityProvider)
 
+	if idp.Enabled.IsFalse() {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "idp disabled")
+	}
+
 	if idp.Status != api.IdentityDriverStatusConnected && idp.Status != api.IdentityDriverStatusDisconnected {
-		return nil, errors.Error(fmt.Sprintf("invalid idp status %s", idp.Status))
+		return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "invalid idp status %s", idp.Status)
 	}
 
 	conf, err := models.GetConfigs(idp, true, nil, nil)

pkg/mcclient/modules/mod_cachedimages.go

@@ -26,7 +26,7 @@ func init() {
 			"OS_Type", "OS_Distribution", "OS_version",
 			"Hypervisor", "Host_count", "Status",
 			"Ref_Count", "cached_count", "image_type",
-			"External_Id",
+			"External_Id", "Is_Public", "Public_Scope", "Tenant", "Tenant_Id",
 		},
 		[]string{})
 

pkg/multicloud/aliyun/image.go

@@ -28,6 +28,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
@@ -57,6 +58,7 @@ const (
 )
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Architecture         string
@@ -129,7 +131,7 @@ func (self *SImage) GetStatus() string {
 	case ImageStatusCreating:
 		return api.CACHED_IMAGE_STATUS_SAVING
 	case ImageStatusAvailable:
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case ImageStatusUnAvailable:
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	case ImageStatusCreateFailed:

pkg/multicloud/aliyun/latitud_and_longitude.go

@@ -41,4 +41,5 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
 	"eu-west-1":      {Latitude: 51.507351, Longitude: -0.127758, City: api.CITY_LONDON, CountryCode: api.COUNTRY_CODE_GB},
 	"me-east-1":      {Latitude: 25.204849, Longitude: 55.270782, City: api.CITY_DUBAI, CountryCode: api.COUNTRY_CODE_AE},
 	"eu-central-1":   {Latitude: 50.110924, Longitude: 8.682127, City: api.CITY_FRANKFURT, CountryCode: api.COUNTRY_CODE_DE},
+	"cn-wulanchabu":  {Latitude: 41.0178065, Longitude: 113.094978, City: api.CITY_WU_LAN_CHA_BU, CountryCode: api.COUNTRY_CODE_CN},
 }

pkg/multicloud/aws/image.go

@@ -28,6 +28,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
 type ImageStatusType string
@@ -74,6 +75,7 @@ type RootDevice struct {
 }
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Architecture string
@@ -185,7 +187,7 @@ func (self *SImage) GetStatus() string {
 	case ImageStatusCreating:
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case ImageStatusAvailable:
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case ImageStatusCreateFailed:
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	default:

pkg/multicloud/azure/image.go

@@ -27,6 +27,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
 type ImageStatusType string
@@ -80,6 +81,7 @@ type ImageProperties struct {
 }
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Properties ImageProperties `json:"properties,omitempty"`
@@ -130,7 +132,7 @@ func (self *SImage) GetStatus() string {
 	case "created":
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case "Succeeded":
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	default:
 		log.Errorf("Unknow image status: %s", self.Properties.ProvisioningState)
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED

pkg/multicloud/ctyun/image.go

@@ -24,6 +24,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
@@ -35,6 +36,7 @@ const (
 
 // http://ctyun-api-url/apiproxy/v3/order/getImages
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	ID        string `json:"id"`
@@ -66,7 +68,7 @@ func (self *SImage) GetGlobalId() string {
 }
 
 func (self *SImage) GetStatus() string {
-	return api.CACHED_IMAGE_STATUS_READY
+	return api.CACHED_IMAGE_STATUS_ACTIVE
 }
 
 func (self *SImage) Refresh() error {

pkg/multicloud/esxi/datacenter.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 
 	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/view"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/types"
 
@@ -237,32 +239,85 @@ func (dc *SDatacenter) getDcObj() *object.Datacenter {
 	return object.NewDatacenter(dc.manager.client.Client, dc.object.Reference())
 }
 
-// fetchVms will identify if VM is a template and return two different arrays; the latter contains all template vms.
-func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]cloudprovider.ICloudVM, []*SVirtualMachine, error) {
-	var vms []mo.VirtualMachine
+func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]*SVirtualMachine, error) {
+	var movms []mo.VirtualMachine
 	if vmRefs != nil {
-		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &vms)
+		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &movms)
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "dc.manager.references2Objects")
+			return nil, errors.Wrap(err, "dc.manager.references2Objects")
 		}
 	}
 
 	// avoid applying new memory and copying
-	retVms := make([]cloudprovider.ICloudVM, 0, len(vms)/2)
-	templateVMs := make([]*SVirtualMachine, 0, 2)
-	for i := 0; i < len(vms); i += 1 {
-		if all || !strings.HasPrefix(vms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
-			vmObj := NewVirtualMachine(dc.manager, &vms[i], dc)
-			if vms[i].Config != nil && vms[i].Config.Template {
-				templateVMs = append(templateVMs, vmObj)
-				continue
-			}
-			if vmObj != nil {
-				retVms = append(retVms, vmObj)
-			}
+	vms := make([]*SVirtualMachine, 0, len(movms))
+	for i := range movms {
+		if all || !strings.HasPrefix(movms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
+			vms = append(vms, NewVirtualMachine(dc.manager, &movms[i], dc))
 		}
 	}
-	return retVms, templateVMs, nil
+	return vms, nil
+}
+
+func (dc *SDatacenter) FetchVMs() ([]*SVirtualMachine, error) {
+	return dc.fetchVMs(property.Filter{})
+}
+
+func (dc *SDatacenter) FetchNoTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = false
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) fetchVMs(filter property.Filter) ([]*SVirtualMachine, error) {
+	odc := dc.getObjectDatacenter()
+	root := odc.Reference()
+	m := view.NewManager(dc.manager.client.Client)
+	v, err := m.CreateContainerView(dc.manager.context, root, []string{"VirtualMachine"}, true)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = v.Destroy(dc.manager.context)
+	}()
+	objs, err := v.Find(dc.manager.context, []string{"VirtualMachine"}, filter)
+	if err != nil {
+		return nil, err
+	}
+	vms, err := dc.fetchVms(objs, false)
+	return vms, err
+}
+
+func (dc *SDatacenter) FetchTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) FetchTemplateVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
+}
+
+func (dc *SDatacenter) FetchVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
 }
 
 func (dc *SDatacenter) fetchDatastores(datastoreRefs []types.ManagedObjectReference) ([]cloudprovider.ICloudStorage, error) {
@@ -381,24 +436,3 @@ func (dc *SDatacenter) GetTemplateVMs() ([]*SVirtualMachine, error) {
 	}
 	return templateVms, nil
 }
-
-func (dc *SDatacenter) GetTemplateVMById(id string) (*SVirtualMachine, error) {
-	id = dc.manager.getPrivateId(id)
-	hosts, err := dc.GetIHosts()
-	if err != nil {
-		return nil, errors.Wrap(err, "SDatacenter.GetIHosts")
-	}
-	for _, ihost := range hosts {
-		host := ihost.(*SHost)
-		tvms, err := host.GetTemplateVMs()
-		if err != nil {
-			return nil, errors.Wrap(err, "host.GetTemplateVMs")
-		}
-		for i := range tvms {
-			if tvms[i].GetGlobalId() == id {
-				return tvms[i], nil
-			}
-		}
-	}
-	return nil, cloudprovider.ErrNotFound
-}

pkg/multicloud/esxi/host.go

@@ -181,6 +181,7 @@ func (self *SHost) fetchVMs(all bool) error {
 	}
 
 	MAX_TRIES := 3
+	var vms []*SVirtualMachine
 	for tried := 0; tried < MAX_TRIES; tried += 1 {
 		hostVms := self.getHostSystem().Vm
 		if len(hostVms) == 0 {
@@ -188,15 +189,20 @@ func (self *SHost) fetchVMs(all bool) error {
 			return nil
 		}
 
-		vms, templatevms, err := dc.fetchVms(hostVms, all)
+		vms, err = dc.fetchVms(hostVms, all)
 		if err != nil {
 			log.Errorf("dc.fetchVms fail %s", err)
 			time.Sleep(time.Second)
 			self.Refresh()
 			continue
 		}
-		self.vms = vms
-		self.tempalteVMs = templatevms
+	}
+	for _, vm := range vms {
+		if vm.IsTemplate() {
+			self.tempalteVMs = append(self.tempalteVMs, vm)
+		} else {
+			self.vms = append(self.vms, vm)
+		}
 	}
 	return nil
 }
@@ -702,9 +708,13 @@ func (self *SHost) CreateVM2(ctx context.Context, ds *SDatastore, params SCreate
 	if err != nil {
 		return nil, errors.Wrap(err, "SEsxiClient.FindHostByIp")
 	}
-	temvm, err := imgHost.GetTemplateVMById(imageInfo.ImageExternalId)
+	dc, err := imgHost.GetDatacenter()
 	if err != nil {
-		return nil, errors.Wrap(err, "SHost.GetTemplateVMById")
+		return nil, errors.Wrap(err, "host.GetDatacenter")
+	}
+	temvm, err := dc.FetchTemplateVMById(imageInfo.ImageExternalId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "datacenter.TemplateVMById for image %q and datacenter %q", imageInfo.ImageExternalId, dc.GetId())
 	}
 	return self.CloneVM(ctx, temvm, ds, params)
 }

pkg/multicloud/esxi/image.go

@@ -26,9 +26,11 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
 type SImage struct {
+	multicloud.SImageBase
 	cache    *SDatastoreImageCache
 	filename string
 	size     int64
@@ -74,12 +76,12 @@ func (self *SImage) GetStatus() string {
 	if err != nil {
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	}
-	return api.CACHED_IMAGE_STATUS_READY
+	return api.CACHED_IMAGE_STATUS_ACTIVE
 }
 
 func (self *SImage) GetImageStatus() string {
 	status := self.GetStatus()
-	if status == api.CACHED_IMAGE_STATUS_READY {
+	if status == api.CACHED_IMAGE_STATUS_ACTIVE {
 		return cloudprovider.IMAGE_STATUS_ACTIVE
 	}
 	return cloudprovider.IMAGE_STATUS_DELETED

pkg/multicloud/esxi/ip.go

@@ -0,0 +1,92 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package esxi
+
+import (
+	"github.com/vmware/govmomi/vim25/mo"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/regutils"
+)
+
+var HOST_PROPS = []string{"name", "config.network", "vm"}
+
+var VM_PROPS = []string{"name", "guest.net", "config.template"}
+
+func (cli *SESXiClient) AllHostIP() (map[string]string, []mo.HostSystem, error) {
+	var hosts []mo.HostSystem
+	err := cli.scanAllMObjects(HOST_PROPS, &hosts)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "scanAllMObjects")
+	}
+	ret := make(map[string]string, len(hosts))
+	for i := range hosts {
+		// find ip
+		host := &SHost{SManagedObject: newManagedObject(cli, &hosts[i], nil)}
+		ip := host.GetAccessIp()
+		ret[host.GetName()] = ip
+	}
+	return ret, hosts, nil
+}
+
+func (cli *SESXiClient) VMIP(host mo.HostSystem) (map[string][]string, error) {
+	var vms []mo.VirtualMachine
+	err := cli.references2Objects(host.Vm, VM_PROPS, &vms)
+	if err != nil {
+		return nil, errors.Wrap(err, "references2Objects")
+	}
+	ret := make(map[string][]string, len(vms))
+	for i := range vms {
+		vm := vms[i]
+		if vm.Config.Template {
+			continue
+		}
+		guestIps := make([]string, 0)
+		for _, net := range vm.Guest.Net {
+			for _, ip := range net.IpAddress {
+				if regutils.MatchIP4Addr(ip) {
+					guestIps = append(guestIps, ip)
+				}
+			}
+		}
+		ret[vm.Name] = guestIps
+	}
+	return ret, nil
+}
+
+func (cli *SESXiClient) VMIP2() (map[string][]string, error) {
+	var vms []mo.VirtualMachine
+	err := cli.scanAllMObjects(VM_PROPS, &vms)
+	if err != nil {
+		return nil, errors.Wrap(err, "scanAllMObjects")
+	}
+	ret := make(map[string][]string, len(vms))
+	for i := range vms {
+		vm := vms[i]
+		if vm.Config.Template {
+			continue
+		}
+		guestIps := make([]string, 0)
+		for _, net := range vm.Guest.Net {
+			for _, ip := range net.IpAddress {
+				if regutils.MatchIP4Addr(ip) {
+					guestIps = append(guestIps, ip)
+				}
+			}
+		}
+		ret[vm.Name] = guestIps
+	}
+	return ret, nil
+}

pkg/multicloud/esxi/manager.go

@@ -31,10 +31,12 @@ import (
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
 	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/sync/errgroup"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -597,3 +599,86 @@ func (cli *SESXiClient) MoveDisk(ctx context.Context, src, dst string, isForce b
 	}
 	return task.Wait(ctx)
 }
+
+var (
+	SIMPLE_HOST_PROPS = []string{"name", "config.network", "vm"}
+	SIMPLE_VM_PROPS   = []string{"name", "guest.net", "config.template"}
+)
+
+type SSimpleVM struct {
+	Name string
+	IPs  []string
+}
+
+func (cli *SESXiClient) HostVmIPs(ctx context.Context) (map[string]string, []SSimpleVM, error) {
+	var hosts []mo.HostSystem
+	err := cli.scanAllMObjects(SIMPLE_HOST_PROPS, &hosts)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "scanAllMObjects")
+	}
+	group, ctx := errgroup.WithContext(ctx)
+	collection := make([][]SSimpleVM, len(hosts))
+	for i := range hosts {
+		j := i
+		group.Go(func() error {
+			vmIps, err := cli.vmIPs(&hosts[j])
+			if err != nil {
+				return err
+			}
+			collection[j] = vmIps
+			return nil
+		})
+	}
+
+	hostIps := make(map[string]string, len(hosts))
+	for i := range hosts {
+		// find ip
+		host := &SHost{SManagedObject: newManagedObject(cli, &hosts[i], nil)}
+		ip := host.GetAccessIp()
+		hostIps[host.GetName()] = ip
+	}
+	err = group.Wait()
+	if err != nil {
+		return nil, nil, err
+	}
+	// length
+	length := 0
+	for i := range collection {
+		length += len(collection[i])
+	}
+	svms := make([]SSimpleVM, 0, length)
+	for i := range collection {
+		for j := range collection[i] {
+			svms = append(svms, collection[i][j])
+		}
+	}
+	return hostIps, svms, nil
+}
+
+func (cli *SESXiClient) vmIPs(host *mo.HostSystem) ([]SSimpleVM, error) {
+	if len(host.Vm) == 0 {
+		return []SSimpleVM{}, nil
+	}
+	var vms []mo.VirtualMachine
+	err := cli.references2Objects(host.Vm, VM_PROPS, &vms)
+	if err != nil {
+		return nil, errors.Wrap(err, "references2Objects")
+	}
+	ret := make([]SSimpleVM, 0, len(vms))
+	for i := range vms {
+		vm := vms[i]
+		if vm.Config.Template {
+			continue
+		}
+		guestIps := make([]string, 0)
+		for _, net := range vm.Guest.Net {
+			for _, ip := range net.IpAddress {
+				if regutils.MatchIP4Addr(ip) {
+					guestIps = append(guestIps, ip)
+				}
+			}
+		}
+		ret = append(ret, SSimpleVM{vm.Name, guestIps})
+	}
+	return ret, nil
+}

pkg/multicloud/esxi/shell/ip.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type IPOption struct {
+		Host bool
+	}
+	shellutils.R(&IPOption{}, "ip-all", "List all ip", func(cli *esxi.SESXiClient, args *IPOption) error {
+		hostIps, _, err := cli.AllHostIP()
+		if err != nil {
+			return err
+		}
+		for name, ip := range hostIps {
+			fmt.Printf("name: %s, ip: %s\n", name, ip)
+		}
+		// for i := range hosts {
+		// 	fmt.Printf("host %s: \n", hosts[i].Name)
+		// 	vmips, err := cli.VMIP(hosts[i])
+		// 	if err != nil {
+		// 		return err
+		// 	}
+		// 	for name, ip := range vmips {
+		// 		fmt.Printf("\tname: %s, ip: %s\n", name, ip)
+		// 	}
+		// }
+		vmips, err := cli.VMIP2()
+		if err != nil {
+			return err
+		}
+		for name, ip := range vmips {
+			fmt.Printf("\tname: %s, ip: %s\n", name, ip)
+		}
+		return nil
+	})
+}

pkg/multicloud/esxi/shell/virtualmachine.go

@@ -27,28 +27,53 @@ import (
 
 func init() {
 	type VirtualMachineListOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		Template bool   `help:"Whether it is tempalte virtual machine"`
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"HostIP"`
+		Template   bool   `help:"Whether it is tempalte virtual machine, default:false"`
 	}
 	shellutils.R(&VirtualMachineListOptions{}, "vm-list", "List vms of a host", func(cli *esxi.SESXiClient, args *VirtualMachineListOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vms, err := host.GetTemplateVMs()
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return err
+			}
+			if args.Template {
+				vms, err := host.GetTemplateVMs()
+				if err != nil {
+					return err
+				}
+				printList(vms, []string{})
+				return nil
+			}
+			vms, err := host.GetIVMs2()
 			if err != nil {
 				return err
 			}
 			printList(vms, []string{})
 			return nil
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			var vms []*esxi.SVirtualMachine
+			if args.Template {
+				vms, err = dc.FetchTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchTemplateVMs")
+				}
+			} else {
+				vms, err = dc.FetchNoTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchNoTemplateVMs")
+				}
+			}
+			printList(vms, []string{})
+			return nil
+		default:
+			return fmt.Errorf("Both Datacenter and HostIP cannot be empty")
 		}
-		vms, err := host.GetIVMs2()
-		if err != nil {
-			return err
-		}
-		printList(vms, []string{})
-		return nil
 	})
 
 	type VirtualMachineCloneOptions struct {
@@ -91,24 +116,45 @@ func init() {
 	})
 
 	type VirtualMachineShowOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		VMID     string `help:"VM ID"`
-		Template bool
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"Host IP"`
+		VMID       string `help:"VM ID"`
+	}
+	getVM := func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) (*esxi.SVirtualMachine, error) {
+		var vm *esxi.SVirtualMachine
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindHostByIp")
+			}
+			ivm, err := host.GetIVMById(args.VMID)
+			if err != nil && errors.Cause(err) != errors.ErrNotFound {
+				return nil, err
+			}
+			if err != nil {
+				vm, err = host.GetTemplateVMById(args.VMID)
+				if err != nil {
+					return nil, errors.Wrap(err, "GetTemplateVMById")
+				}
+			}
+			vm = ivm.(*esxi.SVirtualMachine)
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			vm, err = dc.FetchVMById(args.VMID)
+			if err != nil {
+				return nil, errors.Wrap(err, "FetchVMById")
+			}
+		default:
+			return nil, fmt.Errorf("Both Datacenter and HostIP cannot be empty")
+		}
+		return vm, nil
 	}
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-show", "Show vm details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vm, err := host.GetTemplateVMById(args.VMID)
-			if err != nil {
-				return err
-			}
-			printObject(vm)
-			return nil
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -117,11 +163,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-nics", "Show vm nics details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -134,11 +176,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-disks", "Show vm disks details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -151,17 +189,12 @@ func init() {
 	})
 
 	type VirtualMachineDiskResizeOptions struct {
-		HOSTIP  string `help:"host ip"`
-		VMID    string `help:"virtual machine UUID"`
-		DISKIDX int    `help:"disk index"`
-		SIZEGB  int64  `help:"new size of disk"`
+		VirtualMachineShowOptions
+		DISKIDX int   `help:"disk index"`
+		SIZEGB  int64 `help:"new size of disk"`
 	}
 	shellutils.R(&VirtualMachineDiskResizeOptions{}, "vm-disk-resize", "Resize a vm disk", func(cli *esxi.SESXiClient, args *VirtualMachineDiskResizeOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
 		if err != nil {
 			return err
 		}
@@ -178,11 +211,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -195,15 +224,11 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-file-status", "Show vm files details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
-		vm, err := host.GetIVMById(args.VMID)
-		if err != nil {
-			return err
-		}
-		err = vm.(*esxi.SVirtualMachine).CheckFileInfo(context.Background())
+		err = vm.CheckFileInfo(context.Background())
 		if err != nil {
 			return err
 		}

pkg/multicloud/esxi/storage.go

@@ -259,7 +259,14 @@ func (self *SDatastore) getVMs() ([]cloudprovider.ICloudVM, error) {
 	if len(vms) == 0 {
 		return nil, nil
 	}
-	ret, _, err := dc.fetchVms(vms, false)
+	svms, err := dc.fetchVms(vms, false)
+	if err != nil {
+		return nil, err
+	}
+	ret := make([]cloudprovider.ICloudVM, len(svms))
+	for i := range svms {
+		ret[i] = svms[i]
+	}
 	return ret, err
 }
 

pkg/multicloud/esxi/template.go

@@ -24,9 +24,11 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
 type SVMTemplate struct {
+	multicloud.SImageBase
 	cache *SDatastoreImageCache
 	vm    *SVirtualMachine
 	uuid  string
@@ -65,7 +67,7 @@ func (t *SVMTemplate) GetStatus() string {
 		host := ihost.(*SHost)
 		_, err := host.GetTemplateVMById(t.uuid)
 		if err == nil {
-			return api.CACHED_IMAGE_STATUS_READY
+			return api.CACHED_IMAGE_STATUS_ACTIVE
 		}
 		if errors.Cause(err) != cloudprovider.ErrNotFound {
 			log.Errorf("fail to find templatevm %q: %v", t.uuid, err)
@@ -123,7 +125,7 @@ func (t *SVMTemplate) GetImageType() string {
 
 func (t *SVMTemplate) GetImageStatus() string {
 	status := t.GetStatus()
-	if status == api.CACHED_IMAGE_STATUS_READY {
+	if status == api.CACHED_IMAGE_STATUS_ACTIVE {
 		return cloudprovider.IMAGE_STATUS_ACTIVE
 	}
 	return cloudprovider.IMAGE_STATUS_DELETED

pkg/multicloud/esxi/virtualmachine.go

@@ -43,6 +43,7 @@ import (
 	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/netutils2"
+	"yunion.io/x/onecloud/pkg/util/version"
 )
 
 var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary", "config", "guest", "resourcePool", "layoutEx"}
@@ -74,10 +75,7 @@ func (d byDiskType) Less(i, j int) bool {
 
 func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter) *SVirtualMachine {
 	svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc)}
-	err := svm.fetchHardwareInfo()
-	if err != nil {
-		return nil
-	}
+	svm.fetchHardwareInfo()
 	return svm
 }
 
@@ -580,11 +578,14 @@ func (self *SVirtualMachine) doDetachDisk(ctx context.Context, vdisk *SVirtualDi
 }
 
 func (self *SVirtualMachine) GetVNCInfo() (jsonutils.JSONObject, error) {
-	info, err := self.acquireWebmksTicket("webmks")
-	if err != nil {
-		info, err = self.acquireVmrcUrl()
+	hostVer := self.GetIHost().GetVersion()
+	if version.GE(hostVer, "6.5") {
+		info, err := self.acquireWebmksTicket("webmks")
+		if err == nil {
+			return info, nil
+		}
 	}
-	return info, err
+	return self.acquireVmrcUrl()
 }
 
 func (self *SVirtualMachine) acquireWebmksTicket(ticketType string) (jsonutils.JSONObject, error) {
@@ -606,14 +607,6 @@ func (self *SVirtualMachine) acquireWebmksTicket(ticketType string) (jsonutils.J
 	if port == 0 {
 		port = 443
 	}
-	/*
-		ret.Add(jsonutils.NewString(ticketType), "type")
-		ret.Add(jsonutils.NewString(ticket.Host), "host")
-		ret.Add(jsonutils.NewInt(int64(ticket.Port)), "port")
-		ret.Add(jsonutils.NewString(ticket.Ticket), "ticket")
-		ret.Add(jsonutils.NewString(ticket.SslThumbprint), "slThumbprint")
-		ret.Add(jsonutils.NewString(ticket.CfgFile), "cfgFile")
-	*/
 	url := fmt.Sprintf("wss://%s:%d/ticket/%s", host, port, ticket.Ticket)
 	ret.Add(jsonutils.NewString("wmks"), "protocol")
 	ret.Add(jsonutils.NewString(url), "url")
@@ -710,7 +703,7 @@ func (self *SVirtualMachine) UpdateUserData(userData string) error {
 	return nil
 }
 
-func (self *SVirtualMachine) fetchHardwareInfo() error {
+func (self *SVirtualMachine) fetchHardwareInfo() {
 	self.vnics = make([]SVirtualNIC, 0)
 	self.vdisks = make([]SVirtualDisk, 0)
 	self.cdroms = make([]SVirtualCdrom, 0)
@@ -724,7 +717,8 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	}
 
 	if moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil {
-		return errors.Error("invalid vm config")
+		log.Errorf("invalid vm config, moVM: %v", moVM)
+		return
 	}
 
 	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
@@ -752,7 +746,6 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	sort.Slice(self.vdisks, func(i, j int) bool {
 		return self.vdisks[i].GetIndex() < self.vdisks[j].GetIndex()
 	})
-	return nil
 }
 
 func (self *SVirtualMachine) getVdev(key int32) SVirtualDevice {
@@ -1220,3 +1213,8 @@ func (self *SVirtualMachine) FindMinDiffKey(limit int32) int32 {
 	}
 	return limit
 }
+
+func (self *SVirtualMachine) IsTemplate() bool {
+	movm := self.getVirtualMachine()
+	return movm.Config != nil && movm.Config.Template
+}

pkg/multicloud/google/image.go

@@ -25,6 +25,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
@@ -39,6 +40,7 @@ type SDeprecated struct {
 }
 
 type SImage struct {
+	multicloud.SImageBase
 	storagecache *SStoragecache
 	SResourceBase
 
@@ -120,7 +122,7 @@ func (image *SImage) GetMinRamSizeMb() int {
 func (image *SImage) GetStatus() string {
 	switch image.Status {
 	case "READY":
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case "FAILED":
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	case "PENDING":

pkg/multicloud/google/storagecache.go

@@ -102,7 +102,7 @@ func (cache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.T
 		} else {
 			status := _image.GetStatus()
 			log.Debugf("UploadImage: Image external ID %s exists, status %s", image.ExternalId, status)
-			if status == api.CACHED_IMAGE_STATUS_READY {
+			if status == api.CACHED_IMAGE_STATUS_ACTIVE {
 				return image.ExternalId, nil
 			}
 			err = cache.region.Delete(image.ExternalId)

pkg/multicloud/huawei/image.go

@@ -26,6 +26,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
@@ -50,6 +51,7 @@ const (
 
 // https://support.huaweicloud.com/api-ims/zh-cn_topic_0020091565.html
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Schema                 string    `json:"schema"`
@@ -112,7 +114,7 @@ func (self *SImage) GetStatus() string {
 	case ImageStatusQueued:
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case ImageStatusActive:
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case ImageStatusKilled:
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	default:

pkg/multicloud/image_base.go

@@ -0,0 +1,25 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multicloud
+
+import "yunion.io/x/onecloud/pkg/util/rbacutils"
+
+type SImageBase struct {
+	SVirtualResourceBase
+}
+
+func (self *SImageBase) GetPublicScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeSystem
+}

pkg/multicloud/openstack/image.go

@@ -29,8 +29,10 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 const (
@@ -46,6 +48,7 @@ const (
 )
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Status          string
@@ -157,7 +160,7 @@ func (image *SImage) GetStatus() string {
 	case QUEUED, SAVING, UPLOADING, IMPORTING:
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case ACTIVE:
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case DELETED, DEACTIVATED, PENDING_DELETE, KILLED:
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	default:
@@ -192,6 +195,19 @@ func (image *SImage) GetImageType() string {
 	return cloudprovider.CachedImageTypeSystem
 }
 
+func (image *SImage) GetPublicScope() rbacutils.TRbacScope {
+	switch image.Visibility {
+	case "private":
+		return rbacutils.ScopeNone
+	default:
+		return rbacutils.ScopeSystem
+	}
+}
+
+func (image *SImage) GetProjectId() string {
+	return image.Owner
+}
+
 func (image *SImage) GetSizeByte() int64 {
 	return int64(image.Size)
 }
@@ -246,17 +262,17 @@ func (image *SImage) GetCreatedAt() time.Time {
 }
 
 func (region *SRegion) GetImage(imageId string) (*SImage, error) {
-	images, err := region.GetImages("", "", imageId)
+	resource := "/v2/images/" + imageId
+	resp, err := region.imageGet(resource)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "imageGet(%s)", resource)
 	}
-	if len(images) == 0 {
-		return nil, cloudprovider.ErrNotFound
+	image := &SImage{}
+	err = resp.Unmarshal(image)
+	if err != nil {
+		return nil, errors.Wrapf(err, "resp.Unmarshal")
 	}
-	if len(images) > 1 {
-		return nil, cloudprovider.ErrDuplicateId
-	}
-	return &images[0], nil
+	return image, nil
 }
 
 func (image *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache {

pkg/multicloud/openstack/shell/image.go

@@ -43,11 +43,11 @@ func init() {
 	}
 
 	shellutils.R(&ImageOptions{}, "image-show", "Show image", func(cli *openstack.SRegion, args *ImageOptions) error {
-		image, err := cli.GetImages("", "", args.ID)
+		image, err := cli.GetImage(args.ID)
 		if err != nil {
 			return err
 		}
-		printObject(image[0])
+		printObject(image)
 		return nil
 	})
 

pkg/multicloud/openstack/storagecache.go

@@ -158,7 +158,7 @@ func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.T
 
 	img.storageCache = cache
 
-	return img.Id, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second)
+	return img.Id, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second)
 }
 
 func (cache *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) {

pkg/multicloud/qcloud/image.go

@@ -27,6 +27,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
@@ -42,6 +43,7 @@ const (
 )
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	ImageId            string          //	镜像ID
@@ -140,7 +142,7 @@ func (self *SImage) GetStatus() string {
 	case ImageStatusCreating, ImageStatusSycing, ImageStatusImporting:
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case ImageStatusNormal, ImageStatusUsing:
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	default:
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	}

pkg/multicloud/qcloud/qcloud.go

@@ -581,6 +581,9 @@ func (client *SQcloudClient) getCosClient(bucket *SBucket) (*cos.Client, error)
 					RequestBody:    client.debug,
 					ResponseHeader: client.debug,
 					ResponseBody:   client.debug,
+					Transport: &http.Transport{
+						Proxy: client.cpcfg.ProxyFunc,
+					},
 				},
 			},
 		},

pkg/multicloud/qcloud/storagecache.go

@@ -223,7 +223,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
 	if err != nil {
 		return "", err
 	}
-	err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second)
+	err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second)
 	if err != nil {
 		return "", err
 	}

pkg/multicloud/ucloud/host.go

@@ -258,8 +258,8 @@ func (self *SHost) _createVM(name, imgId string, sysDisk cloudprovider.SDiskInfo
 		log.Errorf("GetImage %s fail %s", imgId, err)
 		return "", err
 	}
-	if img.GetStatus() != api.CACHED_IMAGE_STATUS_READY {
-		log.Errorf("image %s status %s, expect %s", imgId, img.GetStatus(), api.CACHED_IMAGE_STATUS_READY)
+	if img.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE {
+		log.Errorf("image %s status %s, expect %s", imgId, img.GetStatus(), api.CACHED_IMAGE_STATUS_ACTIVE)
 		return "", fmt.Errorf("image not ready")
 	}
 

pkg/multicloud/ucloud/image.go

@@ -23,10 +23,12 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 )
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	Zone             string `json:"Zone"`
@@ -65,7 +67,7 @@ func (self *SImage) GetGlobalId() string {
 func (self *SImage) GetStatus() string {
 	switch self.State {
 	case "Available":
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case "Making":
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case "Unavailable":

pkg/multicloud/ucloud/storagecache.go

@@ -125,7 +125,7 @@ func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			log.Errorf("GetImageStatus error %s", err)
 		}
-		if img.GetStatus() == api.CACHED_IMAGE_STATUS_READY && !isForce {
+		if img.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !isForce {
 			return image.ExternalId, nil
 		}
 	} else {

pkg/multicloud/zstack/image.go

@@ -23,13 +23,13 @@ import (
 	"sort"
 	"time"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/imagetools"
 	"yunion.io/x/onecloud/pkg/util/multipart"
@@ -45,6 +45,7 @@ type SBackupStorageRef struct {
 }
 
 type SImage struct {
+	multicloud.SImageBase
 	storageCache *SStoragecache
 
 	BackupStorageRefs []SBackupStorageRef `json:"backupStorageRefs"`
@@ -117,7 +118,7 @@ func (image *SImage) GetStatus() string {
 	}
 	switch image.Status {
 	case "Ready":
-		return api.CACHED_IMAGE_STATUS_READY
+		return api.CACHED_IMAGE_STATUS_ACTIVE
 	case "Downloading":
 		return api.CACHED_IMAGE_STATUS_CACHING
 	case "Deleted":

pkg/multicloud/zstack/storagecache.go

@@ -105,7 +105,7 @@ func (scache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.
 			log.Errorf("GetImageStatus error %s", err)
 		}
 		status := img.GetStatus()
-		if api.CACHED_IMAGE_STATUS_READY == status && !isForce {
+		if api.CACHED_IMAGE_STATUS_ACTIVE == status && !isForce {
 			return image.ExternalId, nil
 		}
 		log.Debugf("image %s status %s", image.ExternalId, status)
@@ -131,7 +131,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
 		return "", err
 	}
 	img.storageCache = self
-	err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, time.Second*5, time.Minute*20) //windows镜像转换比较慢，等待时间稍微设长一些
+	err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, time.Second*5, time.Minute*20) //windows镜像转换比较慢，等待时间稍微设长一些
 	if err != nil {
 		log.Errorf("waitting for image %s(%s) status ready timeout", img.Name, img.UUID)
 	}

pkg/scheduler/manager/instancegroup_select.go

@@ -98,7 +98,7 @@ func buildHosts(result *core.SchedResultItemList, groups map[string]*models.SGro
 // sortHost sorts the host for guest that is the backup one of the high-availability guest
 // if isBackup is true and the master one if isBackup is false.
 func sortHosts(hosts []*sSchedResultItem, guestInfo *sGuestInfo, isBackup *bool) {
-	sortIndexi, sortIndexj := make([]int64, 4), make([]int64, 4)
+	sortIndexi, sortIndexj := make([]int64, 5), make([]int64, 5)
 	sort.Slice(hosts, func(i, j int) bool {
 		switch {
 		case isBackup == nil:

pkg/vpcagent/models/modelset.go

@@ -526,6 +526,10 @@ func (set Elasticips) NewModel() db.IModel {
 
 func (set Elasticips) AddModel(i db.IModel) {
 	m := i.(*Elasticip)
+	if m.IpAddr == "" || m.NetworkId == "" {
+		// eip records can exist with network id and ip addr not set
+		return
+	}
 	set[m.Id] = m
 }
 

pkg/webconsole/service/service.go

@@ -26,6 +26,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/signalutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/webconsole"
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -66,9 +67,15 @@ func StartService() {
 
 	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, o.OnOptionsChange)
 
+	registerSigTraps()
 	start()
 }
 
+func registerSigTraps() {
+	signalutils.SetDumpStackSignal()
+	signalutils.StartTrap()
+}
+
 func start() {
 	baseOpts := &o.Options.BaseOptions
 	// commonOpts := &o.Options.CommonOptions

pkg/webconsole/session/pty_session.go

@@ -102,6 +102,10 @@ func (p *Pty) Resize(size *pty.Winsize) {
 func (p *Pty) Stop() (err error) {
 	var errs []error
 
+	defer func() {
+		p.Cmd, p.Pty = nil, nil
+	}()
+
 	defer func() {
 		err = errors.NewAggregate(errs)
 	}()
@@ -139,8 +143,5 @@ func (p *Pty) Stop() (err error) {
 		}
 	}()
 
-	defer func() {
-		p.Cmd, p.Pty = nil, nil
-	}()
 	return
 }