Merge pull request #9113 from yousong/automated-cherry-pick-of-#9109-upstream-release-3.3

Automated cherry pick of #9109: fix(host): make sure it won't match vpc guests by accident

.circleci/config.yml

@@ -38,7 +38,7 @@ jobs:
           command: |
             targets="$(circleci tests glob "cmd/*" | grep -v cmd/host-image | circleci tests split)"
             echo $targets | tr ' ' '\n'
-            make -j3 $targets
+            make -j2 $targets
             ls -lh _output/bin/
       - save_cache:
           key: onecloud-build-cache-{{ checksum "go.mod" }}-{{ checksum "Makefile" }}

cmd/redfishcli/main.go

@@ -82,9 +82,11 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 
 func bmcJnlp() {
 	type BmcGetOptions struct {
-		BRAND string `help:"brand of baremetal" choices:"Lenovo|Huawei|HPE|Dell|Supermicro"`
+		BRAND string `help:"brand of baremetal" choices:"Lenovo|Huawei|HPE|Dell|Supermicro|Dell6|Dell7|Dell9"`
 		Save  string `help:"save to file"`
 		Debug bool   `help:"turn on debug mode"`
+		Sku   string `help:"sku"`
+		Model string `help:"model"`
 	}
 	shellutils.R(&BmcGetOptions{}, "bmc-jnlp", "Get Java Console JNLP file", func(args *BmcGetOptions) error {
 		ctx := context.Background()
@@ -98,7 +100,13 @@ func bmcJnlp() {
 		case "hp", "hpe":
 			jnlp, err = bmc.GetIloConsoleJNLP(ctx)
 		case "dell", "dell inc.":
-			jnlp, err = bmc.GetIdracConsoleJNLP(ctx, "", "")
+			jnlp, err = bmc.GetIdracConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell6":
+			jnlp, err = bmc.GetIdrac6ConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell7":
+			jnlp, err = bmc.GetIdrac7ConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell9":
+			jnlp, err = bmc.GetIdrac9ConsoleJNLP(ctx)
 		case "supermicro":
 			jnlp, err = bmc.GetSupermicroConsoleJNLP(ctx)
 		case "lenovo":

go.mod

@@ -131,12 +131,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8
+	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
 	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6
+	yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -1126,8 +1126,10 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8 h1:KQVw5EmohJCqBDMdZ3fAcOQNNMIYgVuNCNWNwg4EM3A=
-yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6 h1:RPpnTSCZwa9VQuWAf7S2dRjnZlYTIM0NH/j5vvUpk9g=
+yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
@@ -1140,7 +1142,7 @@ yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6 h1:k7E8Zjwzyk/lo1dDO0jtLbODwYRNv+w/zsUJrKWCXvo=
-yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8 h1:AkPrqO37Yi3aICys9qDjTlSdVVGh2pWQ8ET+dSJmhoI=
+yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/cachedimage.go

@@ -27,7 +27,7 @@ type CachedImageUncacheImageInput struct {
 }
 
 type CachedimageDetails struct {
-	apis.StandaloneResourceDetails
+	apis.SharableVirtualResourceDetails
 
 	SCachedimage
 

pkg/apis/compute/cachedimages_const.go

@@ -18,7 +18,7 @@ const (
 	CACHED_IMAGE_STATUS_INIT         = "init"
 	CACHED_IMAGE_STATUS_SAVING       = "saving"
 	CACHED_IMAGE_STATUS_CACHING      = "caching"
-	CACHED_IMAGE_STATUS_READY        = "ready"
+	CACHED_IMAGE_STATUS_ACTIVE       = "active"
 	CACHED_IMAGE_STATUS_DELETING     = "deleting"
 	CACHED_IMAGE_STATUS_CACHE_FAILED = "cache_fail"
 

pkg/apis/compute/cloudregions_const.go

@@ -57,6 +57,7 @@ const (
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
 	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
+	CITY_WU_LAN_CHA_BU = "Wulanchabu"  //乌兰察布
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京

pkg/apis/compute/dbinstance.go

@@ -350,3 +350,8 @@ type DBInstanceFilterListInput struct {
 
 	VpcFilterListInput
 }
+
+type DBInstanceNetworkListInput struct {
+	apis.VirtualJointResourceBaseListInput
+	DBInstanceFilterListInput
+}

pkg/apis/compute/elasticip.go

@@ -21,23 +21,22 @@ type SElasticipCreateInput struct {
 
 	// 区域名称或Id, 建议使用Id
 	// 在指定区域内创建弹性公网ip
-	Cloudregion string `json:"cloudregion"`
+	CloudregionId string `json:"cloudregion_id"`
+	// swagger:ignore
+	Cloudregion string `json:"cloudregion" yunion-deprecated-by:"cloudregion_id"`
+	// swagger:ignore
+	Region string `json:"region" yunion-deprecated-by:"cloudregion_id"`
+	// swagger:ignore
+	RegionId string `json:"region_id" yunion-deprecated-by:"cloudregion_id"`
 
 	// swagger:ignore
-	Region string
+	Cloudprovider string `json:"cloudprovider" yunion-deprecated-by:"manager_id"`
 	// swagger:ignore
-	RegionId string
-	// swagger:ignore
-	CloudregionId string
-
+	Manager string `json:"manager" yunion-deprecated-by:"manager_id"`
 	// 子订阅Id, 建议使用Id
 	// 使用指定子订阅创建弹性公网ip
 	// 弹性公网ip和虚拟机在同一区域同一子订阅底下，才可以进行绑定操作
-	Cloudprovider string `json:"cloudprovider"`
-	// swagger:ignore
-	Manager string
-	// swagger:ignore
-	ManagerId string
+	ManagerId string `json:"manager_id"`
 
 	// 计费类型: 流量或带宽
 	//
@@ -63,11 +62,11 @@ type SElasticipCreateInput struct {
 
 	Mode string `json:"mode"`
 
-	// 子网名称或Id
-	// 私有云创建此参数必传,例如Openstack, ZStack
-	Network string `json:"network"`
 	// swagger:ignore
-	NetworkId string
+	Network string `json:"network" yunion-deprecated-by:"network_id"`
+	// 子网Id
+	// 私有云创建此参数必传,例如Openstack, ZStack
+	NetworkId string `json:"network_id"`
 }
 
 type ElasticipDetails struct {

pkg/apis/compute/input.go

@@ -49,7 +49,7 @@ type KeypairListInput struct {
 }
 
 type CachedimageListInput struct {
-	apis.StandaloneResourceListInput
+	apis.SharableVirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput

pkg/apis/compute/loadbalancer.go

@@ -201,7 +201,10 @@ type LoadbalancerResourceInfo struct {
 	// 可用区ID
 	ZoneId string `json:"zone_id"`
 
-	ZoneResourceInfoBase
+	ZoneResourceInfo
+
+	// cloud provider info
+	ManagedResourceInfo
 }
 
 type LoadbalancerResourceInput struct {

pkg/apis/compute/secgroup.go

@@ -248,3 +248,6 @@ type GuestsecgroupDetails struct {
 	// 安全组名称
 	Secgroup string `json:"secgroup"`
 }
+
+type SecurityGroupPurgeInput struct {
+}

pkg/apis/compute/storage_const.go

@@ -81,6 +81,7 @@ const (
 	STORAGE_GOOGLE_LOCAL_SSD   = "local-ssd"   //本地SSD暂存盘 (最多8个)
 	STORAGE_GOOGLE_PD_STANDARD = "pd-standard" //标准永久性磁盘
 	STORAGE_GOOGLE_PD_SSD      = "pd-ssd"      //SSD永久性磁盘
+	STORAGE_GOOGLE_PD_BALANCED = "pd-balanced" //平衡永久性磁盘
 
 	// ctyun storage type
 	STORAGE_CTYUN_SSD  = "SSD"  // 超高IO云硬盘

pkg/apis/image/image.go

@@ -88,3 +88,12 @@ type ImageCreateInput struct {
 	// 镜像属性
 	Properties map[string]string `json:"properties"`
 }
+
+type ImageUpdateStatusInput struct {
+	apis.Meta
+
+	// 镜像状态
+	Status string `json:"status"`
+	// 更新镜像状态原因
+	Reason string `json:"reason"`
+}

pkg/baremetal/manager.go

@@ -61,6 +61,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/dhcp"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
@@ -1398,7 +1399,15 @@ func (b *SBaremetalInstance) DoRedfishPowerOn() error {
 	b.ClearSSHConfig()
 	redfishApi := b.GetRedfishCli(ctx)
 	if redfishApi != nil {
-		return redfishApi.Reset(ctx, "On")
+		err := redfishApi.Reset(ctx, "On")
+		if err != nil {
+			if httputils.ErrorCode(err) == 409 {
+				log.Warningf("redfishApi.Reset On fail %s", err)
+			} else {
+				return errors.Wrap(err, "redfishApi.Reset On")
+			}
+		}
+		return nil
 	}
 	return fmt.Errorf("Baremetal %s redfishApi is nil", b.GetId())
 }
@@ -1950,7 +1959,7 @@ func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error)
 	case "hp", "hpe":
 		return bmc.GetIloConsoleJNLP(ctx)
 	case "dell", "dell inc.":
-		return bmc.GetIdracConsoleJNLP(ctx, "", "")
+		return bmc.GetIdracConsoleJNLP(ctx, b.GetSerialNumber(), b.GetModel())
 	case "supermicro":
 		return bmc.GetSupermicroConsoleJNLP(ctx)
 	case "lenovo":

pkg/baremetal/pxe/dhcp.go

@@ -269,6 +269,9 @@ func (req *dhcpRequest) findNetworkConf(session *mcclient.ClientSession, filterU
 		params.Add(jsonutils.NewString(
 			fmt.Sprintf("guest_dhcp.endswith(',%s')", req.RelayAddr)),
 			"filter.3")
+		params.Add(jsonutils.NewString(
+			fmt.Sprintf("guest_dhcp.equals(%s)", req.RelayAddr)),
+			"filter.4")
 		params.Add(jsonutils.JSONTrue, "filter_any")
 	}
 	params.Add(jsonutils.JSONTrue, "is_on_premise")

pkg/cloudcommon/db/caller.go

@@ -302,6 +302,7 @@ func FetchCustomizeColumns(
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
+		// NOTE: don't use obj update jsonDict as retval
 		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
 		retVal[i] = jsonDict
 	}

pkg/cloudcommon/db/db_dispatcher.go

@@ -1351,7 +1351,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 		ret := make([]sCreateResult, len(multiData))
 		for i := range multiData {
 			var model IModel
-			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i)
+			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i+1)
 			if err == nil {
 				ret[i] = sCreateResult{model: model, err: nil}
 			} else {

pkg/cloudcommon/db/domain.go

@@ -107,12 +107,7 @@ func (manager *SDomainizedResourceBaseManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	if len(query.ProjectDomains) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.ProjectDomains {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.ProjectDomains)
 		tenants := TenantCacheManager.GetDomainQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/jointbase.go

@@ -34,7 +34,7 @@ import (
 type SJointResourceBase struct {
 	SResourceBase
 
-	RowId int64 `primary:"true" auto_increment:"true"`
+	RowId int64 `primary:"true" auto_increment:"true" list:"user"`
 }
 
 type SJointResourceBaseManager struct {

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/db/opslog.go

@@ -288,31 +288,29 @@ type SOpsLogManager struct {
 type SOpsLog struct {
 	SModelBase
 
-	Id      int64  `primary:"true" auto_increment:"true" list:"user"`                                        // = Column(BigInteger, primary_key=True)
-	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`               // = Column(VARCHAR(40, charset='ascii'), nullable=False)
-	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
-	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`               //= Column(VARCHAR(128, charset='utf8'), nullable=False)
-	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`                //= Column(VARCHAR(32, charset='ascii'), nullable=False)
+	Id      int64  `primary:"true" auto_increment:"true" list:"user"`
+	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
+	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`
+	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`
 	Notes   string `charset:"utf8" list:"user" create:"required"`
 
-	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"required" index:"true"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"required"`                  // tenant    = Column(VARCHAR(128, charset='utf8'))
+	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"optional" index:"true"`
+	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"required"`
-	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"required"`
+	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
+	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	User     string `width:"128" charset:"utf8" list:"user" create:"required"`  // = Column(VARCHAR(128, charset='utf8'))
+	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"`
+	User     string `width:"128" charset:"utf8" list:"user" create:"required"`
 	DomainId string `width:"128" charset:"ascii" list:"user" create:"optional"`
 	Domain   string `width:"128" charset:"utf8" list:"user" create:"optional"`
-	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(64, charset='ascii'))
+	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"`
 
-	// BillingType    string    `width:"64" charset:"ascii" default:"postpaid" list:"user" create:"user"`      // billing_type = Column(VARCHAR(64, charset='ascii'), nullable=True)
-	OpsTime time.Time `nullable:"false" list:"user"` // = Column(DateTime, nullable=False)
+	OpsTime time.Time `nullable:"false" list:"user"`
 
 	OwnerDomainId  string `name:"owner_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
-	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	// owner_user_id   = Column(VARCHAR(ID_LENGTH, charset='ascii'))
+	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"`
 }
 
 var OpsLog *SOpsLogManager
@@ -677,6 +675,7 @@ func (manager *SOpsLogManager) ValidateCreateData(ctx context.Context,
 	query jsonutils.JSONObject,
 	data apis.OpsLogCreateInput,
 ) (apis.OpsLogCreateInput, error) {
+	data.User = ownerId.GetUserName()
 	return data, nil
 }
 

pkg/cloudcommon/db/project.go

@@ -102,12 +102,7 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 	}
 	if len(query.Projects) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.Projects {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.Projects)
 		tenants := TenantCacheManager.GetTenantQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/quotas/register.go

@@ -33,6 +33,7 @@ var (
 func init() {
 	quotaManagerTable = make(map[reflect.Type]IQuotaManager)
 
+	db.AddUsages = AddUsages
 	db.CancelUsages = CancelUsages
 	db.CancelPendingUsagesInContext = cancelPendingUsagesInContext
 	db.InitPendingUsagesInContext = initPendingUsagesInContext

pkg/cloudcommon/db/usages.go

@@ -28,6 +28,8 @@ type IUsage interface {
 var (
 	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
 
+	AddUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
+
 	CancelPendingUsagesInContext func(ctx context.Context, userCred mcclient.TokenCredential) error
 
 	InitPendingUsagesInContext func(ctx context.Context) context.Context

pkg/cloudcommon/db/virtualresource.go

@@ -393,11 +393,21 @@ func (model *SVirtualResourceBase) AllowPerformCancelDelete(ctx context.Context,
 func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if model.PendingDeleted && !model.Deleted {
 		err := model.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "model.DoCancelPendingDelete")
+		}
+		model.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
 
+func (model *SVirtualResourceBase) RecoverUsages(ctx context.Context, userCred mcclient.TokenCredential) {
+	usages := model.GetIModel().GetUsages()
+	if AddUsages != nil && len(usages) > 0 {
+		AddUsages(ctx, userCred, usages)
+	}
+}
+
 func (model *SVirtualResourceBase) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	err := model.CancelPendingDelete(ctx, userCred)
 	if err == nil {
@@ -416,7 +426,10 @@ func (model *SVirtualResourceBase) GetIVirtualModel() IVirtualModel {
 
 func (model *SVirtualResourceBase) CancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	if model.PendingDeleted && !model.Deleted {
-		return model.MarkCancelPendingDelete(ctx, userCred)
+		err := model.MarkCancelPendingDelete(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "MarkCancelPendingDelete")
+		}
 	}
 	return nil
 }

pkg/cloudcommon/notifyclient/notify.go

@@ -126,7 +126,7 @@ func RawNotify(recipientId []string, isGroup bool, channel notify.TNotifyChannel
 	msg.Topic = topic
 	body, _ := getContent(event, "content", channel, data)
 	if len(body) == 0 {
-		body = data.String()
+		body, _ = data.GetString()
 	}
 	msg.Msg = body
 	// log.Debugf("send notification %s %s", topic, body)

pkg/cloudprovider/resources.go

@@ -156,7 +156,7 @@ type ICloudZone interface {
 }
 
 type ICloudImage interface {
-	ICloudResource
+	IVirtualResource
 
 	Delete(ctx context.Context) error
 	GetIStoragecache() ICloudStoragecache
@@ -173,6 +173,7 @@ type ICloudImage interface {
 	GetImageFormat() string
 	GetCreatedAt() time.Time
 	UEFI() bool
+	GetPublicScope() rbacutils.TRbacScope
 }
 
 type ICloudStoragecache interface {

pkg/cloudprovider/securitygroup.go

@@ -23,6 +23,7 @@ import (
 )
 
 const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
+const DEFAULT_LOCAL_RULE_ID = "default_local_rule_id"
 
 type SecurityGroupFilterOptions struct {
 	VpcId     string
@@ -44,6 +45,42 @@ type SecurityRule struct {
 	ExternalId string
 }
 
+type LocalSecurityRule struct {
+	secrules.SecurityRule
+	ExternalId string
+}
+
+func (r LocalSecurityRule) String() string {
+	return r.SecurityRule.String()
+}
+
+type LocalSecurityRuleSet []LocalSecurityRule
+
+func (srs LocalSecurityRuleSet) Len() int {
+	return len(srs)
+}
+
+func (srs LocalSecurityRuleSet) Swap(i, j int) {
+	srs[i], srs[j] = srs[j], srs[i]
+}
+
+func (srs LocalSecurityRuleSet) Less(i, j int) bool {
+	if srs[i].Priority > srs[j].Priority {
+		return true
+	} else if srs[i].Priority == srs[j].Priority {
+		return srs[i].String() < srs[j].String()
+	}
+	return false
+}
+
+func (srs LocalSecurityRuleSet) AllowList() secrules.SecurityRuleSet {
+	rules := secrules.SecurityRuleSet{}
+	for _, r := range srs {
+		rules = append(rules, r.SecurityRule)
+	}
+	return rules.AllowList()
+}
+
 type TPriorityOrder int
 
 var (
@@ -88,9 +125,6 @@ func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr
 		defaultRule.Priority = max
 	}
 	defaultRule.Priority -= int(order)
-	if onlyAllowRules {
-		defaultRule.Priority = -1
-	}
 	return append(rules, defaultRule)
 }
 
@@ -112,13 +146,15 @@ func CompareRules(
 	defaultInRule, defaultOutRule SecurityRule,
 	onlyAllowRules bool, debug bool,
 ) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
-	localInRules := secrules.SecurityRuleSet{}
-	localOutRules := secrules.SecurityRuleSet{}
+	localInRules := LocalSecurityRuleSet{}
+	localOutRules := LocalSecurityRuleSet{}
 	for i := range localRules {
+		localRule := LocalSecurityRule{}
+		localRule.SecurityRule = localRules[i]
 		if localRules[i].Direction == secrules.DIR_IN {
-			localInRules = append(localInRules, localRules[i])
+			localInRules = append(localInRules, localRule)
 		} else {
-			localOutRules = append(localOutRules, localRules[i])
+			localOutRules = append(localOutRules, localRule)
 		}
 	}
 	inRules := SecurityRuleSet{}
@@ -132,17 +168,22 @@ func CompareRules(
 	}
 	var inCommon, outCommon = inRules, outRules
 
-	defaultLocalInRule := *secrules.MustParseSecurityRule("in:deny any")
-	defaultLocalOutRule := *secrules.MustParseSecurityRule("out:allow any")
+	defaultLocalInRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
+	defaultLocalInRule.SecurityRule = *secrules.MustParseSecurityRule("in:deny any")
+	defaultLocalOutRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
+	defaultLocalOutRule.SecurityRule = *secrules.MustParseSecurityRule("out:allow any")
 
 	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
 	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
 
+	defaultInEquals, defaultOutEquals := true, true
 	if defaultLocalInRule.String() != defaultInRule.String() {
 		localInRules = append(localInRules, defaultLocalInRule)
+		defaultInEquals = false
 	}
 	if defaultLocalOutRule.String() != defaultOutRule.String() {
 		localOutRules = append(localOutRules, defaultLocalOutRule)
+		defaultOutEquals = false
 	}
 
 	sort.Sort(localInRules)
@@ -150,9 +191,26 @@ func CompareRules(
 
 	localInAllowList := localInRules.AllowList()
 	localOutAllowList := localOutRules.AllowList()
+	_localInRules := LocalSecurityRuleSet{}
+	for i := range localInAllowList {
+		rule := LocalSecurityRule{}
+		rule.SecurityRule = localInAllowList[i]
+		_localInRules = append(_localInRules, rule)
+	}
+	_localOutRules := LocalSecurityRuleSet{}
+	for i := range localOutAllowList {
+		rule := LocalSecurityRule{}
+		rule.SecurityRule = localOutAllowList[i]
+		_localOutRules = append(_localOutRules, rule)
+	}
 	if onlyAllowRules {
-		localInRules = localInAllowList
-		localOutRules = localOutAllowList
+		localOutRules, localInRules = _localOutRules, _localInRules
+	}
+	if len(_localInRules) < len(localInRules) {
+		localInRules = _localInRules
+	}
+	if len(_localOutRules) < len(localOutRules) {
+		localOutRules = _localOutRules
 	}
 
 	SortSecurityRule(inRules, order, onlyAllowRules)
@@ -199,7 +257,7 @@ func CompareRules(
 		return init
 	}
 
-	var compare = func(localRules secrules.SecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
+	var compare = func(localRules LocalSecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
 		i, j, inc, prePriority := 0, 0, 1, 0
 		for i < len(localRules) || j < len(remoteRules) {
 			if i < len(localRules) && j < len(remoteRules) {
@@ -214,7 +272,11 @@ func CompareRules(
 					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
 						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
 					}
-					common = append(common, remoteRules[j])
+					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+						common = append(common, remoteRules[j])
+					}
 					i++
 					j++
 				} else if cmp < 0 {
@@ -225,7 +287,11 @@ func CompareRules(
 				} else {
 					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
 					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-					add = append(add, SecurityRule{SecurityRule: localRules[i]})
+					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+						add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
+					}
 					i++
 					inc++
 				}
@@ -241,7 +307,11 @@ func CompareRules(
 				}
 				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
 				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-				add = append(add, SecurityRule{SecurityRule: localRules[i]})
+				if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+					(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+					(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+					add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
+				}
 				i++
 				inc++
 			}
@@ -250,17 +320,17 @@ func CompareRules(
 	}
 
 	type rulePair struct {
-		localRules  []secrules.SecurityRule
+		localRules  LocalSecurityRuleSet
 		remoteRules []SecurityRule
 		protocol    string
 	}
 
-	var splitRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) []rulePair {
+	var splitRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) []rulePair {
 		rules := map[string]rulePair{}
 		for _, r := range localRules {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
 			}
 			pair.localRules = append(pair.localRules, r)
 			rules[r.Protocol] = pair
@@ -269,7 +339,7 @@ func CompareRules(
 		for _, r := range remoteRules {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
 			}
 			pair.remoteRules = append(pair.remoteRules, r)
 			rules[r.Protocol] = pair
@@ -282,7 +352,7 @@ func CompareRules(
 		return ret
 	}
 
-	var compareRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
+	var compareRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
 		pairs := splitRules(localRules, remoteRules)
 		for _, r := range pairs {
 			_common, _add, _dels := compare(r.localRules, r.remoteRules)

pkg/compute/guestdrivers/base.go

@@ -314,6 +314,10 @@ func (self *SBaseGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
 	return false
 }
 
+func (self *SBaseGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return true
+}
+
 func (self *SBaseGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_CONFIG
 }

pkg/compute/guestdrivers/esxi.go

@@ -172,7 +172,7 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 	if img.ImageType != cloudprovider.CachedImageTypeSystem {
 		return desc
 	}
-	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_READY).SubQuery()
+	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE).SubQuery()
 	scQ := models.StoragecacheManager.Query().In("id", sciSubQ)
 	storageCaches := make([]models.SStoragecache, 0, 1)
 	err = db.FetchModelObjects(models.StoragecacheManager, scQ, &storageCaches)
@@ -368,3 +368,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/guestdrivers/google.go

@@ -77,6 +77,7 @@ func (self *SGoogleGuestDriver) GetStorageTypes() []string {
 	return []string{
 		api.STORAGE_GOOGLE_PD_SSD,
 		api.STORAGE_GOOGLE_PD_STANDARD,
+		api.STORAGE_GOOGLE_PD_BALANCED,
 		api.STORAGE_GOOGLE_LOCAL_SSD,
 	}
 }
@@ -113,7 +114,7 @@ func (self *SGoogleGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	if !utils.IsInStringArray(guest.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status)
 	}
-	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD}) {
+	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED}) {
 		return fmt.Errorf("Cannot resize %s disk", storage.StorageType)
 	}
 	return nil
@@ -132,13 +133,15 @@ func (self *SGoogleGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		minGB := -1
 		maxGB := -1
 		switch disk.Backend {
-		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 			minGB = 10
 			maxGB = 65536
 		case api.STORAGE_GOOGLE_LOCAL_SSD:
 			minGB = 375
 			maxGB = 375
 			localDisk++
+		default:
+			return nil, httperrors.NewInputParameterError("Unknown google storage type %s", disk.Backend)
 		}
 		if i == 0 && disk.Backend == api.STORAGE_GOOGLE_LOCAL_SSD {
 			return nil, httperrors.NewInputParameterError("System disk does not support %s disk", disk.Backend)

pkg/compute/guestdrivers/managedvirtual.go

@@ -435,8 +435,8 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte
 	}
 
 	iVM, err := func() (cloudprovider.ICloudVM, error) {
-		lockman.LockObject(ctx, host)
-		defer lockman.ReleaseObject(ctx, host)
+		lockman.LockObject(ctx, guest)
+		defer lockman.ReleaseObject(ctx, guest)
 
 		iVM, err := ihost.CreateVM(&desc)
 		if err != nil {

pkg/compute/guestdrivers/qcloud.go

@@ -226,6 +226,10 @@ func (self *SQcloudGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_RUNNING
 }
 
+func (self *SQcloudGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return false
+}
+
 func (self *SQcloudGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }

pkg/compute/hostdrivers/google.go

@@ -48,7 +48,7 @@ func (self *SGoogleHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb
 	minGB := 10
 	maxGB := -1
 	switch storage.StorageType {
-	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 		maxGB = 65536
 	default:
 		return fmt.Errorf("Not support resize %s disk", storage.StorageType)

pkg/compute/hostdrivers/managedvirtual.go

@@ -182,7 +182,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		osType, _ := params.GetString("properties", "os_type")
 
 		scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId, "")
-		if scimg.Status != api.CACHED_IMAGE_STATUS_READY {
+		if scimg.Status != api.CACHED_IMAGE_STATUS_ACTIVE {
 			scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host")
 		}
 		iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, "")
@@ -208,7 +208,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		if err := snapshot.Delete(); err != nil {
 			log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err)
 		}
-		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_READY, "")
+		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_ACTIVE, "")
 		return result, nil
 	})
 	return nil

pkg/compute/models/buckets.go

@@ -199,7 +199,13 @@ func (manager *SBucketManager) newFromCloudBucket(
 
 	stats := extBucket.GetStats()
 	bucket.SizeBytes = stats.SizeBytes
+	if bucket.SizeBytes < 0 {
+		bucket.SizeBytes = 0
+	}
 	bucket.ObjectCnt = stats.ObjectCount
+	if bucket.ObjectCnt < 0 {
+		bucket.ObjectCnt = 0
+	}
 
 	limit := extBucket.GetLimit()
 	limitSupport := extBucket.LimitSupport()
@@ -259,7 +265,13 @@ func (bucket *SBucket) syncWithCloudBucket(
 	diff, err := db.UpdateWithLock(ctx, bucket, func() error {
 		stats := extBucket.GetStats()
 		bucket.SizeBytes = stats.SizeBytes
+		if bucket.SizeBytes < 0 {
+			bucket.SizeBytes = 0
+		}
 		bucket.ObjectCnt = stats.ObjectCount
+		if bucket.ObjectCnt < 0 {
+			bucket.ObjectCnt = 0
+		}
 
 		if !statsOnly {
 			limit := extBucket.GetLimit()
@@ -704,11 +716,7 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	prefix := input.Prefix
 	isRecursive := false
@@ -782,11 +790,7 @@ func (bucket *SBucket) PerformTempUrl(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second)
 	if err != nil {
@@ -830,11 +834,7 @@ func (bucket *SBucket) PerformMakedir(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	_, err = cloudprovider.GetIObject(iBucket, key+"/")
@@ -901,11 +901,7 @@ func (bucket *SBucket) PerformDelete(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 	ok := jsonutils.NewDict()
 	results := modulebase.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
@@ -965,11 +961,7 @@ func (bucket *SBucket) PerformUpload(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, appParams.Request.Header)
@@ -1007,7 +999,7 @@ func (bucket *SBucket) PerformUpload(
 		if inc.SizeBytes < 0 {
 			inc.SizeBytes = 0
 		}
-	} else if err == cloudprovider.ErrNotFound {
+	} else if errors.Cause(err) == cloudprovider.ErrNotFound {
 		// new upload
 		inc.SizeBytes = sizeBytes
 		inc.ObjectCount = 1
@@ -1173,11 +1165,7 @@ func (bucket *SBucket) PerformSync(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, statsOnly)
@@ -1221,11 +1209,7 @@ func (bucket *SBucket) GetDetailsAcl(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	objKey := input.Key
 	var acl cloudprovider.TBucketACLType
@@ -1234,7 +1218,7 @@ func (bucket *SBucket) GetDetailsAcl(
 	} else {
 		object, err := cloudprovider.GetIObject(iBucket, objKey)
 		if err != nil {
-			if err == cloudprovider.ErrNotFound {
+			if errors.Cause(err) == cloudprovider.ErrNotFound {
 				return output, httperrors.NewNotFoundError("object %s not found", objKey)
 			} else {
 				return output, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err)
@@ -1309,11 +1293,7 @@ func (bucket *SBucket) PerformLimit(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = iBucket.SetLimit(limit)
@@ -1426,11 +1406,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, nil, errors.Wrap(err, "GetIBucket")
 	}
 	objects := make([]cloudprovider.ICloudObject, 0)
 	for _, key := range input.Key {
@@ -1443,7 +1419,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 		} else {
 			object, err := cloudprovider.GetIObject(iBucket, key)
 			if err != nil {
-				if err == cloudprovider.ErrNotFound {
+				if errors.Cause(err) == cloudprovider.ErrNotFound {
 					return nil, nil, httperrors.NewResourceNotFoundError("object %s not found", key)
 				} else {
 					return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObject error %s", err)

pkg/compute/models/cachedimages.go

@@ -37,11 +37,12 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 type SCachedimageManager struct {
-	db.SStandaloneResourceBaseManager
+	db.SSharableVirtualResourceBaseManager
 	db.SExternalizedResourceBaseManager
 }
 
@@ -49,7 +50,7 @@ var CachedimageManager *SCachedimageManager
 
 func init() {
 	CachedimageManager = &SCachedimageManager{
-		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
 			SCachedimage{},
 			"cachedimages_tbl",
 			"cachedimage",
@@ -60,7 +61,7 @@ func init() {
 }
 
 type SCachedimage struct {
-	db.SStandaloneResourceBase
+	db.SSharableVirtualResourceBase
 	db.SExternalizedResourceBase
 
 	// 镜像大小单位: Byte
@@ -100,10 +101,10 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {
 	if cnt > 0 {
 		return httperrors.NewNotEmptyError("The image has been cached on storages")
 	}
-	if self.GetStatus() == "active" && !self.isReferenceSessionExpire() {
+	if self.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !self.isReferenceSessionExpire() {
 		return httperrors.NewConflictError("the image reference session has not been expired!")
 	}
-	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
 func (self *SCachedimage) isReferenceSessionExpire() bool {
@@ -194,32 +195,44 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
-	imgId, _ := info.GetString("id")
-	if len(imgId) == 0 {
+	img := struct {
+		Id          string
+		Size        int64
+		Name        string
+		Status      string
+		IsPublic    bool
+		ProjectId   string `json:"tenant_id"`
+		PublicScope string
+	}{}
+	err := info.Unmarshal(&img)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Unmarshal %s", info.String())
+	}
+	if len(img.Id) == 0 {
 		return nil, fmt.Errorf("invalid image info")
 	}
+	if len(img.Name) == 0 {
+		img.Name = img.Id
+	}
 
 	imageCache := SCachedimage{}
 	imageCache.SetModelManager(manager, &imageCache)
 
-	size, _ := info.Int("size")
-	name, _ := info.GetString("name")
-	if len(name) == 0 {
-		name = imgId
-	}
-
-	name, err := db.GenerateName(manager, nil, name)
+	img.Name, err = db.GenerateName(manager, nil, img.Name)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "db.GenerateName(%s)", img.Name)
 	}
 
-	err = manager.RawQuery().Equals("id", imgId).First(&imageCache)
+	err = manager.RawQuery().Equals("id", img.Id).First(&imageCache)
 	if err != nil {
 		if err == sql.ErrNoRows { // insert
-			imageCache.Id = imgId
-			imageCache.Name = name
-			imageCache.Size = size
+			imageCache.Id = img.Id
+			imageCache.Name = img.Name
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 
 			err = manager.TableSpec().Insert(ctx, &imageCache)
@@ -230,13 +243,16 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 
 			return &imageCache, nil
 		} else {
-			log.Errorf("fetching image cache (%s) failed: %s", imgId, err)
+			log.Errorf("fetching image cache (%s) failed: %s", img.Id, err)
 			return nil, err
 		}
 	} else { // update
 		diff, err := db.Update(&imageCache, func() error {
-			imageCache.Size = size
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 			if imageCache.Deleted == true {
 				imageCache.Deleted = false
@@ -323,16 +339,15 @@ func (manager *SCachedimageManager) FetchCustomizeColumns(
 	isList bool,
 ) []api.CachedimageDetails {
 	rows := make([]api.CachedimageDetails, len(objs))
-	stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	virtRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range rows {
 		ci := objs[i].(*SCachedimage)
 		rows[i] = api.CachedimageDetails{
-			StandaloneResourceDetails: stdRows[i],
-			Status:                    ci.GetStatus(),
-			OsType:                    ci.GetOSType(),
-			OsDistribution:            ci.GetOSDistribution(),
-			OsVersion:                 ci.GetOSVersion(),
-			Hypervisor:                ci.GetHypervisor(),
+			SharableVirtualResourceDetails: virtRows[i],
+			OsType:                         ci.GetOSType(),
+			OsDistribution:                 ci.GetOSDistribution(),
+			OsVersion:                      ci.GetOSVersion(),
+			Hypervisor:                     ci.GetHypervisor(),
 		}
 		rows[i].CachedCount, _ = ci.getStoragecacheCount()
 	}
@@ -372,7 +387,7 @@ func (self *SCachedimage) PerformUncacheImage(ctx context.Context, userCred mccl
 }
 
 func (self *SCachedimage) addRefCount() {
-	if self.GetStatus() != "active" {
+	if self.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE {
 		return
 	}
 	_, err := db.Update(self, func() error {
@@ -397,7 +412,7 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 		Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))).
 		Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)).
-		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_READY)).
+		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE)).
 		Filter(sqlchemy.Equals(host.Field("status"), api.HOST_STATUS_RUNNING)).
 		Filter(sqlchemy.IsTrue(host.Field("enabled"))).
 		Filter(sqlchemy.Equals(host.Field("host_status"), api.HOST_ONLINE)).
@@ -455,12 +470,17 @@ func (self *SCachedimage) canDeleteLastCache() bool {
 	return false
 }
 
-func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error {
+func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = image.GetName()
 		self.Size = image.GetSizeByte()
 		self.ExternalId = image.GetGlobalId()
 		self.ImageType = image.GetImageType()
+		self.PublicScope = string(image.GetPublicScope())
+		self.Status = image.GetStatus()
+		if image.GetPublicScope() == rbacutils.ScopeSystem {
+			self.IsPublic = true
+		}
 		self.UEFI = tristate.NewFromBool(image.UEFI())
 		sImage := cloudprovider.CloudImage2Image(image)
 		self.Info = jsonutils.Marshal(&sImage)
@@ -468,10 +488,12 @@ func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mccli
 		return nil
 	})
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
+
+	SyncCloudProject(userCred, self, ownerId, image, managerId)
 	return err
 }
 
-func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) (*SCachedimage, error) {
+func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) (*SCachedimage, error) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -491,12 +513,21 @@ func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userC
 	cachedImage.LastSync = time.Now().UTC()
 	cachedImage.ImageType = image.GetImageType()
 	cachedImage.ExternalId = image.GetGlobalId()
+	cachedImage.Status = image.GetStatus()
+	cachedImage.PublicScope = string(image.GetPublicScope())
+	switch image.GetPublicScope() {
+	case rbacutils.ScopeNone:
+	default:
+		cachedImage.IsPublic = true
+	}
 
 	err = manager.TableSpec().Insert(ctx, &cachedImage)
 	if err != nil {
 		return nil, err
 	}
 
+	SyncCloudProject(userCred, &cachedImage, ownerId, image, managerId)
+
 	return &cachedImage, nil
 }
 
@@ -529,7 +560,7 @@ func (image *SCachedimage) requestRefreshExternalImage(ctx context.Context, user
 		log.Errorf("iCache.GetIImageById fail %s", err)
 		return nil, err
 	}
-	err = image.syncWithCloudImage(ctx, userCred, iImage)
+	err = image.syncWithCloudImage(ctx, userCred, nil, iImage, "")
 	if err != nil {
 		log.Errorf("image.syncWithCloudImage fail %s", err)
 		return nil, err
@@ -549,7 +580,7 @@ func (image *SCachedimage) getValidStoragecache() []SStoragecache {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 
 	caches := make([]SStoragecache, 0)
 	err := db.FetchModelObjects(StoragecacheManager, q, &caches)
@@ -589,7 +620,7 @@ func (image *SCachedimage) GetUsableZoneIds() ([]string, error) {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 	q = q.Filter(sqlchemy.Equals(q.Field("status"), api.ZONE_ENABLE))
 
 	result := []string{}
@@ -632,6 +663,10 @@ func (manager *SCachedimageManager) ListItemFilter(
 	query api.CachedimageListInput,
 ) (*sqlchemy.SQuery, error) {
 	var err error
+	q, err = manager.SSharableBaseResourceManager.ListItemFilter(ctx, q, userCred, query.SharableResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
+	}
 
 	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
 		cachedImages := CachedimageManager.Query().SubQuery()
@@ -647,9 +682,9 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
 	}
 
-	q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
 	}
 
 	q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput)
@@ -667,7 +702,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -682,7 +717,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -704,9 +739,9 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.OrderByExtraFields")
 	}
 
 	return q, nil
@@ -715,10 +750,44 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 func (manager *SCachedimageManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
+	q, err = manager.SSharableVirtualResourceBaseManager.QueryDistinctExtraField(q, field)
 	if err == nil {
 		return q, nil
 	}
 
 	return q, httperrors.ErrNotFound
 }
+
+func (manager *SCachedimageManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
+	q, err := manager.SSharableVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableVirtualResourceBaseManager.L.ListItemExportKeys")
+	}
+	return q, nil
+}
+
+func (manager *SCachedimageManager) InitializeData() error {
+	images := []SCachedimage{}
+	q := manager.Query().IsNullOrEmpty("tenant_id")
+	err := db.FetchModelObjects(manager, q, &images)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range images {
+		_, err := db.Update(&images[i], func() error {
+			images[i].IsPublic = true
+			images[i].PublicScope = string(rbacutils.ScopeSystem)
+			images[i].ProjectId = "system"
+			if len(images[i].ExternalId) > 0 {
+				images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE
+			} else {
+				images[i].Status = images[i].GetStatus()
+			}
+			return nil
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update(%s)", images[i].Id)
+		}
+	}
+	return nil
+}

pkg/compute/models/cloudaccounts.go

@@ -48,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -355,7 +356,14 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		return output, httperrors.NewNotSupportedError("not support for cloudaccount with provider '%s'", input.Provider)
 	}
 	// validate first
-	input.CloudaccountCreateInput, err = scm.ValidateCreateData(ctx, userCred, userCred, query, input.CloudaccountCreateInput)
+	ownerId, err := scm.FetchOwnerId(ctx, jsonutils.Marshal(input))
+	if err != nil {
+		return output, errors.Wrap(err, "FetchOwnerId in PerformPrepareNets")
+	}
+	if ownerId == nil {
+		ownerId = userCred
+	}
+	input.CloudaccountCreateInput, err = scm.ValidateCreateData(ctx, userCred, ownerId, query, input.CloudaccountCreateInput)
 	if err != nil {
 		return output, err
 	}
@@ -417,9 +425,11 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 	if err != nil {
 		return output, errors.Wrap(err, "provider.GetOnPremiseIRegion")
 	}
-	hosts, err := iregion.GetIHosts()
+	// hack
+	client := iregion.(*esxi.SESXiClient)
+	hostIps, simpleVms, err := client.HostVmIPs(ctx)
 	if err != nil {
-		return output, errors.Wrap(err, "unable to get hosts")
+		return output, errors.Wrap(err, "unable to fetch ips of hosts and vms")
 	}
 	// fetch networks
 	networks := make([][]SNetwork, len(wires))
@@ -430,20 +440,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		networks[i] = nets
 	}
-	// fetch host ips
+
 	// key of ipHosts is host's ip
-	ipHosts := make(map[netutils.IPV4Addr]cloudprovider.ICloudHost, len(hosts))
-	for i := range hosts {
-		ip := hosts[i].GetAccessIp()
-		if len(ip) == 0 {
-			log.Errorf("unable to get accessip of host %q", hosts[i].GetName())
-			continue
-		}
+	ipHosts := make(map[netutils.IPV4Addr]string, len(hostIps))
+	for name, ip := range hostIps {
 		addr, err := netutils.NewIPV4Addr(ip)
 		if err != nil {
 			return output, err
 		}
-		ipHosts[addr] = hosts[i]
+		ipHosts[addr] = name
 	}
 	// Find suitable wire and the network containing the Host IP in suitable wire.
 	var (
@@ -490,15 +495,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Give the suggested network configuration for the Host IP that does not have a corresponding suitable network.
 	noNetHostIP := make([]netutils.IPV4Addr, 0, len(ipHosts))
-	for hip, host := range ipHosts {
+	for ip, name := range ipHosts {
 		rnet := api.CAHostNet{
-			Name: host.GetName(),
-			IP:   host.GetAccessIp(),
+			Name: name,
+			IP:   ip.String(),
 		}
-		if net, ok := suitableNetworks[hip]; ok {
+		if net, ok := suitableNetworks[ip]; ok {
 			rnet.SuitableNetwork = net.GetId()
 		} else {
-			noNetHostIP = append(noNetHostIP, hip)
+			noNetHostIP = append(noNetHostIP, ip)
 		}
 		output.Hosts = append(output.Hosts, rnet)
 	}
@@ -515,7 +520,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Find the suitable network containing the VM IP in Project 'input.Project', and if not, give the corresponding suggested network configuration in this project.
 	project := input.Project
-	nets := []*SNetwork{}
+	excludeNets := []*SNetwork{}
 	var allNets []SNetwork
 	if suitableWire != nil {
 		allNets, err = suitableWire.getNetworks(userCred, rbacutils.ScopeSystem)
@@ -524,85 +529,164 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		for i := range allNets {
 			if allNets[i].ProjectId == project {
-				nets = append(nets, &allNets[i])
+				excludeNets = append(excludeNets, &allNets[i])
 			}
 		}
 	}
-	// find suitable network
-	vms := make([]cloudprovider.ICloudVM, 0, len(hosts))
-	for _, host := range hosts {
-		vs, err := host.GetIVMs()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable to get VMs of host %q", host.GetName())
-		}
-		vms = append(vms, vs...)
+
+	type vm struct {
+		FakeID int
+		IP     netutils.IPV4Addr
+		Name   string
 	}
-	ipVMs := make(map[netutils.IPV4Addr]int, len(vms))
-	for i, vm := range vms {
-		nics, err := vm.GetINics()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable get nics of vm %q", vm.GetName())
-		}
-		for _, nic := range nics {
-			ipStr := nic.GetIP()
-			if len(ipStr) == 0 {
-				continue
-			}
-			ip, err := netutils.NewIPV4Addr(ipStr)
-			if err != nil {
-				return output, errors.Wrapf(err, "unable to new IPV4Addr for ip %q", nic.GetIP())
-			}
-			ipVMs[ip] = i
-		}
-		output.Guests = append(output.Guests, api.CAGuestNet{
-			Name:   vm.GetName(),
-			IPNets: make([]api.CAIPNet, 0, 1),
-		})
-	}
-	suitableNetworks = make(map[netutils.IPV4Addr]*SNetwork)
-	if len(nets) != 0 {
-		for vip := range ipVMs {
-			for i := range nets {
-				ipRange := nets[i].GetIPRange()
-				if !ipRange.Contains(vip) {
-					continue
+	vms := make([]vm, 0, len(simpleVms))
+	var nip netutils.IPV4Addr
+	guestMap := make(map[int]*api.CAGuestNet)
+	for i := range simpleVms {
+		id := i
+		if len(simpleVms[i].IPs) == 0 {
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   simpleVms[i].Name,
+					IPNets: []api.CAIPNet{},
 				}
-				suitableNetworks[vip] = nets[i]
-				break
 			}
 		}
-	}
-
-	noNetVMIPs := make([]netutils.IPV4Addr, 0, len(ipVMs))
-	for vip, i := range ipVMs {
-		ipnet := api.CAIPNet{
-			IP: vip.String(),
+		for _, ip := range simpleVms[i].IPs {
+			nip, err = netutils.NewIPV4Addr(ip)
+			if err != nil {
+				return output, err
+			}
+			vms = append(vms, vm{
+				FakeID: i,
+				IP:     nip,
+				Name:   simpleVms[i].Name,
+			})
 		}
-		if net, ok := suitableNetworks[vip]; ok {
-			ipnet.SuitableNetwork = net.GetId()
-		} else {
-			noNetVMIPs = append(noNetVMIPs, vip)
-		}
-		output.Guests[i].IPNets = append(output.Guests[ipVMs[vip]].IPNets, ipnet)
 	}
+	// sort vms via vm's ip
+	sort.Slice(vms, func(i, j int) bool {
+		return vms[i].IP < vms[j].IP
+	})
 
-	// excludedIRs is to prevent network conflicts
-	excludedIRs := make([]netutils.IPV4AddrRange, len(allNets))
-	for i := range excludedIRs {
-		excludedIRs[i] = allNets[i].GetIPRange()
+	excludeNets = make([]*SNetwork, 0, len(allNets)+len(output.HostSuggestedNetworks))
+	for i := range allNets {
+		excludeNets = append(excludeNets, &allNets[i])
 	}
 	for i := range output.HostSuggestedNetworks {
-		startIPStr := output.HostSuggestedNetworks[i].GuestIpStart
-		endIPStr := output.HostSuggestedNetworks[i].GuestIpEnd
-		startIP, _ := netutils.NewIPV4Addr(startIPStr)
-		endIP, _ := netutils.NewIPV4Addr(endIPStr)
-		excludedIRs = append(excludedIRs, netutils.NewIPV4AddrRange(startIP, endIP))
+		startipStr := output.HostSuggestedNetworks[i].GuestIpStart
+		endipStr := output.HostSuggestedNetworks[i].GuestIpEnd
+		excludeNets = append(excludeNets, &SNetwork{GuestIpEnd: endipStr, GuestIpStart: startipStr})
 	}
-	if len(noNetVMIPs) > 0 {
-		sConfs := scm.suggestVMNetwors(noNetVMIPs, excludedIRs)
-		confs := make([]api.CANetConf, len(sConfs))
+	// sort excludeNets via their GuestIpStart
+	sort.Slice(excludeNets, func(i, j int) bool {
+		return excludeNets[i].GuestIpStart < excludeNets[j].GuestIpStart
+	})
+
+	svNets := make([]netutils.IPV4AddrRange, 0, 5)
+	var vmi, neti int
+
+	lastEndIp := netutils.IPV4Addr(0)
+Loop:
+	for neti = 0; neti < len(excludeNets); {
+		if vmi >= len(vms) {
+			break
+		}
+		startIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpStart)
+		endIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpEnd)
+		switch {
+		case vms[vmi].IP > endIp:
+			lastEndIp = endIp
+			neti++
+		case vms[vmi].IP >= startIp:
+			for vms[vmi].IP <= endIp {
+				id := vms[vmi].FakeID
+				if _, ok := guestMap[id]; !ok {
+					guestMap[id] = &api.CAGuestNet{
+						Name:   vms[vmi].Name,
+						IPNets: []api.CAIPNet{},
+					}
+				}
+				guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+					IP:              vms[vmi].IP.String(),
+					SuitableNetwork: excludeNets[neti].Id,
+				})
+				vmi += 1
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+			lastEndIp = endIp
+			neti++
+		default:
+			for vms[vmi].IP < startIp {
+				suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+				if suggestStartIp <= lastEndIp {
+					suggestStartIp = lastEndIp + 1
+				}
+				if suggestStartIp >= startIp {
+					break
+				}
+				suggestEndIp := suggestStartIp.NetAddr(24) + 255
+				if suggestEndIp >= startIp {
+					suggestEndIp = startIp - 1
+				}
+				svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+				for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+					id := vms[vmi].FakeID
+					if _, ok := guestMap[id]; !ok {
+						guestMap[id] = &api.CAGuestNet{
+							Name:   vms[vmi].Name,
+							IPNets: []api.CAIPNet{},
+						}
+					}
+					guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+						IP: vms[vmi].IP.String(),
+					})
+					vmi++
+				}
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+		}
+	}
+
+	for vmi < len(vms) {
+		suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+		if suggestStartIp <= lastEndIp {
+			suggestStartIp = lastEndIp + 1
+		}
+		suggestEndIp := suggestStartIp.NetAddr(24) + 255
+		svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+		for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+			id := vms[vmi].FakeID
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   vms[vmi].Name,
+					IPNets: []api.CAIPNet{},
+				}
+			}
+			guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+				IP: vms[vmi].IP.String(),
+			})
+			vmi++
+		}
+	}
+
+	for _, guest := range guestMap {
+		output.Guests = append(output.Guests, *guest)
+	}
+
+	if len(svNets) > 0 {
+		confs := make([]api.CANetConf, len(svNets))
 		for i := range confs {
-			confs[i].CASimpleNetConf = sConfs[i]
+			confs[i].CASimpleNetConf = api.CASimpleNetConf{
+				GuestIpStart: svNets[i].StartIp().String(),
+				GuestIpEnd:   svNets[i].EndIp().String(),
+				GuestIpMask:  24,
+				GuestGateway: (svNets[i].StartIp().NetAddr(24) + 1).String(),
+			}
 			confs[i].Name = fmt.Sprintf("%s-guest-network-%d", input.Name, i+1)
 		}
 		output.GuestSuggestedNetworks = confs

pkg/compute/models/cloudproviders.go

@@ -1394,8 +1394,7 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	params := jsonutils.NewDict()
 	task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
-		log.Errorf("%s", err)
-		return err
+		return errors.Wrapf(err, "NewTask")
 	}
 	self.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartCloudproviderDeleteTask")
 	task.ScheduleRun(nil)

pkg/compute/models/cloudregions.go

@@ -117,6 +117,22 @@ func (self *SCloudregion) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudregion) GetElasticIps(managerId, eipMode string) ([]SElasticip, error) {
+	q := ElasticipManager.Query().Equals("cloudregion_id", self.Id)
+	if len(managerId) > 0 {
+		q = q.Equals("manager_id", managerId)
+	}
+	if len(eipMode) > 0 {
+		q = q.Equals("mode", eipMode)
+	}
+	eips := []SElasticip{}
+	err := db.FetchModelObjects(ElasticipManager, q, &eips)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return eips, nil
+}
+
 func (self *SCloudregion) GetZoneQuery() *sqlchemy.SQuery {
 	zones := ZoneManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {

pkg/compute/models/dbinstance_accounts.go

@@ -730,3 +730,21 @@ func (manager *SDBInstanceAccountManager) ListItemExportKeys(ctx context.Context
 	}
 	return q, nil
 }
+
+func (manager *SDBInstanceAccountManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	accounts := []SDBInstanceAccount{}
+	err := db.FetchModelObjects(manager, q, &accounts)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range accounts {
+		err = accounts[i].Purge(context.Background(), nil)
+		if err != nil {
+			return errors.Wrapf(err, "purge %s", accounts[i].Id)
+		}
+	}
+	log.Debugf("SDBInstanceAccountManager cleaned %d dirty data.", len(accounts))
+	return nil
+}

pkg/compute/models/dbinstance_databases.go

@@ -486,3 +486,21 @@ func (manager *SDBInstanceDatabaseManager) ListItemExportKeys(ctx context.Contex
 	}
 	return q, nil
 }
+
+func (manager *SDBInstanceDatabaseManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	databases := []SDBInstanceDatabase{}
+	err := db.FetchModelObjects(manager, q, &databases)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range databases {
+		err = databases[i].Purge(context.Background(), nil)
+		if err != nil {
+			return errors.Wrapf(err, "purge %s", databases[i].Id)
+		}
+	}
+	log.Debugf("SDBInstanceDatabaseManager cleaned %d dirty data.", len(databases))
+	return nil
+}

pkg/compute/models/dbinstance_skus.go

@@ -164,6 +164,9 @@ func (manager *SDBInstanceSkuManager) ListItemFilter(
 	if len(query.EngineVersion) > 0 {
 		q = q.In("engine_version", query.EngineVersion)
 	}
+	if len(query.Providers) > 0 {
+		q = q.In("provider", query.Providers)
+	}
 
 	for k, zoneIds := range map[string][]string{"zone1": query.Zone1, "zone2": query.Zone2, "zone3": query.Zone3} {
 		ids := []string{}

pkg/compute/models/dbinstancenetworks.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/netutils"
@@ -81,6 +82,24 @@ func (self *SDBInstanceNetwork) AllowDeleteItem(ctx context.Context, userCred mc
 	return false
 }
 
+// RDS网络列表
+func (manager *SDBInstanceNetworkManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.DBInstanceNetworkListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SVirtualJointResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualJointResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SVirtualJointResourceBase.ListItemFilter")
+	}
+	q, err = manager.SDBInstanceResourceBaseManager.ListItemFilter(ctx, q, userCred, query.DBInstanceFilterListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SDBInstanceResourceBaseManager.ListItemFilter")
+	}
+	return q, nil
+}
+
 func (self *SDBInstanceNetwork) GetNetwork() (*SNetwork, error) {
 	network, err := NetworkManager.FetchById(self.NetworkId)
 	if err != nil {
@@ -229,3 +248,23 @@ func (manager *SDBInstanceNetworkManager) newFromCloudDBNetwork(ctx context.Cont
 	}
 	return nil
 }
+
+func (manager *SDBInstanceNetworkManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	networks := []SDBInstanceNetwork{}
+	err := db.FetchModelObjects(manager, q, &networks)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range networks {
+		_, err := db.Update(&networks[i], func() error {
+			return networks[i].MarkDelete()
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update.MarkDelete")
+		}
+	}
+	log.Debugf("SDBInstanceNetworkManager cleaned %d dirty data.", len(networks))
+	return nil
+}

pkg/compute/models/dbinstances.go

@@ -1351,17 +1351,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 }
 
 func (self *SDBInstance) syncRemoveCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential) error {
-	lockman.LockObject(ctx, self)
-	defer lockman.ReleaseObject(ctx, self)
-
-	self.SetDisableDelete(userCred, false)
-
-	err := self.ValidateDeleteCondition(ctx)
-	if err != nil { // cannot delete
-		self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
-		return errors.Wrap(err, "ValidateDeleteCondition")
-	}
-	return self.RealDelete(ctx, userCred)
+	return self.Purge(ctx, userCred)
 }
 
 func (self *SDBInstance) ValidateDeleteCondition(ctx context.Context) error {
@@ -1546,7 +1536,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extInstance.GetBillingType()
-			self.ExpiredAt = extInstance.GetExpiredAt()
+			if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extInstance.IsAutoRenew()
 		}
 
@@ -1636,7 +1628,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 
@@ -1652,17 +1646,31 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	return &instance, nil
 }
 
+type SRdsCountStat struct {
+	TotalRdsCount  int
+	TotalCpuCount  int
+	TotalMemSizeMb int
+}
+
 func (man *SDBInstanceManager) TotalCount(
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel,
 	providers []string, brands []string, cloudEnv string,
-) (int, error) {
-	q := man.Query()
+) (SRdsCountStat, error) {
+	sq := man.Query().SubQuery()
+	q := sq.Query(sqlchemy.COUNT("total_rds_count"),
+		sqlchemy.SUM("total_cpu_count", sq.Field("vcpu_count")),
+		sqlchemy.SUM("total_mem_size_mb", sq.Field("vmem_size_mb")))
+
 	q = scopeOwnerIdFilter(q, scope, ownerId)
 	q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv)
 	q = RangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id"), nil, nil)
-	return q.CountWithError()
+
+	stat := SRdsCountStat{}
+	row := q.Row()
+	err := q.Row2Struct(row, &stat)
+	return stat, err
 }
 
 func (dbinstance *SDBInstance) GetQuotaKeys() quotas.IQuotaKeys {

pkg/compute/models/disks.go

@@ -1537,7 +1537,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		disk.BillingType = extDisk.GetBillingType()
-		disk.ExpiredAt = extDisk.GetExpiredAt()
+		if expired := extDisk.GetExpiredAt(); !expired.IsZero() {
+			disk.ExpiredAt = expired
+		}
 		disk.AutoRenew = extDisk.IsAutoRenew()
 	}
 
@@ -2142,9 +2144,12 @@ func (self *SDisk) AllowPerformCancelDelete(ctx context.Context, userCred mcclie
 }
 
 func (self *SDisk) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
@@ -2563,13 +2568,18 @@ func (self *SDisk) PerformChangeOwner(ctx context.Context, userCred mcclient.Tok
 	}
 	for i := range snapshots {
 		snapshot := snapshots[i]
-		lockman.LockObject(ctx, &snapshot)
-		_, err := snapshot.PerformChangeOwner(ctx, userCred, query, input)
+		err := func() error {
+			lockman.LockObject(ctx, &snapshot)
+			defer lockman.ReleaseObject(ctx, &snapshot)
+			_, err := snapshot.PerformChangeOwner(ctx, userCred, query, input)
+			if err != nil {
+				return err
+			}
+			return nil
+		}()
 		if err != nil {
-			lockman.ReleaseObject(ctx, &snapshot)
 			return nil, errors.Wrapf(err, "fail to change owner of this disk(%s)'s snapshot %s", self.Id, snapshot.Id)
 		}
-		lockman.ReleaseObject(ctx, &snapshot)
 	}
 	return nil, nil
 }

pkg/compute/models/elasticcache_instances.go

@@ -509,6 +509,8 @@ func (self *SElasticcache) syncRemoveCloudElasticcache(ctx context.Context, user
 
 	self.SetDisableDelete(userCred, false)
 
+	self.DeleteSubResources(ctx, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
 		self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
@@ -647,7 +649,9 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/elasticips.go

@@ -29,6 +29,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
@@ -247,19 +248,6 @@ func (manager *SElasticipManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi
 	return q, httperrors.ErrNotFound
 }
 
-func (manager *SElasticipManager) getEipsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SElasticip, error) {
-	eips := make([]SElasticip, 0)
-	q := manager.Query().Equals("cloudregion_id", region.Id)
-	if provider != nil {
-		q = q.Equals("manager_id", provider.Id)
-	}
-	err := db.FetchModelObjects(manager, q, &eips)
-	if err != nil {
-		return nil, err
-	}
-	return eips, nil
-}
-
 func (self *SElasticip) GetRegion() *SCloudregion {
 	return CloudregionManager.FetchRegionById(self.CloudregionId)
 }
@@ -328,7 +316,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 	// remoteEips := make([]cloudprovider.ICloudEIP, 0)
 	syncResult := compare.SyncResult{}
 
-	dbEips, err := manager.getEipsByRegion(region, provider)
+	dbEips, err := region.GetElasticIps(provider.Id, api.EIP_MODE_STANDALONE_EIP)
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -426,8 +414,11 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m
 			case api.EIP_ASSOCIATE_TYPE_SERVER:
 				sq := HostManager.Query().SubQuery()
 				return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("host_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), self.ManagerId))
-			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY, api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
+			case api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
 				return q.Equals("manager_id", self.ManagerId)
+			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY:
+				sq := VpcManager.Query("id").Equals("manager_id", self.ManagerId)
+				return q.In("vpc_id", sq.SubQuery())
 			}
 			return q
 		})
@@ -474,7 +465,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.
 		factory, _ := provider.GetProviderFactory()
 		if factory != nil && factory.IsSupportPrepaidResources() {
 			self.BillingType = ext.GetBillingType()
-			self.ExpiredAt = ext.GetExpiredAt()
+			if expired := ext.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = ext.IsAutoRenew()
 		}
 
@@ -795,51 +788,20 @@ func (manager *SElasticipManager) getEipByExtEip(ctx context.Context, userCred m
 	return manager.newFromCloudEip(ctx, userCred, extEip, provider, region, syncOwnerId)
 }
 
-func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.SElasticipCreateInput) (*jsonutils.JSONDict, error) {
-	var (
-		region   *SCloudregion
-		provider *SCloudprovider
-		err      error
-	)
-	for _, cloudregion := range []string{input.Cloudregion, input.Region, input.RegionId} {
-		if len(cloudregion) > 0 {
-			input.Cloudregion = cloudregion
-			break
-		}
+func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.SElasticipCreateInput) (api.SElasticipCreateInput, error) {
+	if input.CloudregionId == "" {
+		input.CloudregionId = api.DEFAULT_REGION_ID
 	}
-	if input.Cloudregion == "" {
-		input.Cloudregion = api.DEFAULT_REGION_ID
-	}
-	if obj, err := CloudregionManager.FetchByIdOrName(nil, input.Cloudregion); err != nil {
+	obj, err := CloudregionManager.FetchByIdOrName(nil, input.CloudregionId)
+	if err != nil {
 		if err != sql.ErrNoRows {
-			return nil, httperrors.NewGeneralError(err)
-		} else {
-			return nil, httperrors.NewResourceNotFoundError("Region %s not found", input.Cloudregion)
+			return input, httperrors.NewGeneralError(err)
 		}
-	} else {
-		region = obj.(*SCloudregion)
+		return input, httperrors.NewResourceNotFoundError2("cloudregion", input.CloudregionId)
 	}
+	region := obj.(*SCloudregion)
 	input.CloudregionId = region.GetId()
 
-	for _, cloudprovider := range []string{input.Cloudprovider, input.Manager, input.ManagerId} {
-		if len(cloudprovider) > 0 {
-			input.Cloudprovider = cloudprovider
-			break
-		}
-	}
-	if input.Cloudprovider != "" {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(nil, input.Cloudprovider)
-		if err != nil {
-			if err != sql.ErrNoRows {
-				return nil, httperrors.NewGeneralError(err)
-			} else {
-				return nil, httperrors.NewResourceNotFoundError("Cloud provider %s not found", input.Cloudprovider)
-			}
-		}
-		provider = providerObj.(*SCloudprovider)
-		input.ManagerId = provider.Id
-	}
-
 	// publicIp cannot be created standalone
 	input.Mode = api.EIP_MODE_STANDALONE_EIP
 
@@ -848,15 +810,30 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 	}
 
 	if !utils.IsInStringArray(input.ChargeType, []string{api.EIP_CHARGE_TYPE_BY_BANDWIDTH, api.EIP_CHARGE_TYPE_BY_TRAFFIC}) {
-		return nil, httperrors.NewInputParameterError("charge type %s not supported", input.ChargeType)
+		return input, httperrors.NewInputParameterError("charge type %s not supported", input.ChargeType)
 	}
 
-	if input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput); err != nil {
-		return nil, err
+	input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
+	if err != nil {
+		return input, err
 	}
 
-	if err = region.GetDriver().ValidateCreateEipData(ctx, userCred, &input); err != nil {
-		return nil, err
+	err = region.GetDriver().ValidateCreateEipData(ctx, userCred, &input)
+	if err != nil {
+		return input, err
+	}
+
+	var provider *SCloudprovider = nil
+	if input.ManagerId != "" {
+		providerObj, err := CloudproviderManager.FetchByIdOrName(nil, input.ManagerId)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return input, httperrors.NewGeneralError(err)
+			}
+			return input, httperrors.NewResourceNotFoundError2("cloudprovider", input.ManagerId)
+		}
+		provider = providerObj.(*SCloudprovider)
+		input.ManagerId = provider.Id
 	}
 
 	//避免参数重名后还有pending.eip残留
@@ -864,10 +841,10 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 	quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, region, provider)
 	eipPendingUsage.SetKeys(quotaKeys)
 	if err = quotas.CheckSetPendingQuota(ctx, userCred, eipPendingUsage); err != nil {
-		return nil, err
+		return input, err
 	}
 
-	return input.JSON(input), nil
+	return input, nil
 }
 
 func (eip *SElasticip) GetQuotaKeys() (quotas.IQuotaKeys, error) {
@@ -1246,8 +1223,8 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		chargeType = api.EIP_CHARGE_TYPE_BY_TRAFFIC
 	}
 
-	eip := SElasticip{}
-	eip.SetModelManager(manager, &eip)
+	eip := &SElasticip{}
+	eip.SetModelManager(manager, eip)
 
 	eip.Mode = api.EIP_MODE_STANDALONE_EIP
 	// do not implicitly auto dellocate EIP, should be set by user explicitly
@@ -1265,7 +1242,9 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		hostq := HostManager.Query().SubQuery()
 		wireq := WireManager.Query().SubQuery()
 		hostwireq := HostwireManager.Query().SubQuery()
+		scope := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), NetworkManager.KeywordPlural(), policy.PolicyActionList)
 		q := NetworkManager.Query()
+		q = NetworkManager.FilterByOwner(q, userCred, scope)
 		q = q.Join(wireq, sqlchemy.Equals(wireq.Field("id"), q.Field("wire_id")))
 		q = q.Join(hostwireq, sqlchemy.Equals(hostwireq.Field("wire_id"), wireq.Field("id")))
 		q = q.Join(hostq, sqlchemy.Equals(hostq.Field("id"), host.Id))
@@ -1275,17 +1254,21 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 			return nil, errors.Wrapf(err, "fetch eip networks usable in host %s(%s)",
 				host.Name, host.Id)
 		}
+		var net *SNetwork
 		for i := range nets {
-			net := &nets[i]
+			net = &nets[i]
 			cnt, err := net.GetFreeAddressCount()
 			if err != nil {
 				continue
 			}
 			if cnt > 0 {
-				eip.NetworkId = net.Id
 				break
 			}
 		}
+		if net == nil {
+			return nil, errors.Error("no usable eip network")
+		}
+		eip.NetworkId = net.Id
 	}
 
 	var err error
@@ -1294,11 +1277,12 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		return nil, errors.Wrap(err, "db.GenerateName")
 	}
 
-	err = manager.TableSpec().Insert(ctx, &eip)
+	err = manager.TableSpec().Insert(ctx, eip)
 	if err != nil {
 		log.Errorf("create EIP record fail %s", err)
 		return nil, err
 	}
+	db.OpsLog.LogEvent(eip, db.ACT_CREATE, eip.GetShortDesc(ctx), userCred)
 
 	eipPendingUsage := &SRegionQuota{Eip: 1}
 	keys := fetchRegionalQuotaKeys(
@@ -1310,7 +1294,7 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 	eipPendingUsage.SetKeys(keys)
 	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage, true)
 
-	return &eip, nil
+	return eip, nil
 }
 
 func (eip *SElasticip) AllocateAndAssociateVM(ctx context.Context, userCred mcclient.TokenCredential, vm *SGuest, parentTaskId string) error {

pkg/compute/models/external_projects.go

@@ -26,11 +26,15 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -198,6 +202,25 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
 		self.Status = ext.GetStatus()
+		if self.DomainId == account.DomainId && account.AutoCreateProject && options.Options.EnableAutoRenameProject {
+			tenant, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
+			if err != nil {
+				return errors.Wrapf(err, "TenantCacheManager.FetchTenantById(%s)", self.ProjectId)
+			}
+			if tenant.Name != self.Name {
+				s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+				params := map[string]string{"name": self.Name}
+				_, err := modules.Projects.Update(s, tenant.Id, jsonutils.Marshal(params))
+				if err != nil {
+					return errors.Wrapf(err, "update project name from %s -> %s", tenant.Name, self.Name)
+				}
+				_, err = db.Update(tenant, func() error {
+					tenant.Name = self.Name
+					return nil
+				})
+				return err
+			}
+		}
 		if self.DomainId != account.DomainId {
 			self.ProjectId = account.ProjectId
 			self.DomainId = account.DomainId
@@ -205,8 +228,7 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 		return nil
 	})
 	if err != nil {
-		log.Errorf("SyncWithCloudProject fail %s", err)
-		return err
+		return errors.Wrapf(err, "db.UpdateWithLock")
 	}
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
 	return nil

pkg/compute/models/guest_actions.go

@@ -2023,6 +2023,12 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T
 			}
 			return nil, httperrors.NewBadRequestError(err.Error())
 		}
+		if _, err := db.Update(&ngn[0], func() error {
+			ngn[0].EipId = gn.EipId
+			return nil
+		}); err != nil {
+			return nil, err
+		}
 
 		return ngn, nil
 	}()
@@ -2587,9 +2593,12 @@ func (model *SGuest) AllowPerformCancelDelete(ctx context.Context, userCred mccl
 }
 
 func (self *SGuest) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "DoCancelPendingDelete")
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
@@ -3635,9 +3644,11 @@ func (self *SGuest) SaveRenewInfo(
 	guestdisks := self.GetDisks()
 	for i := 0; i < len(guestdisks); i += 1 {
 		disk := guestdisks[i].GetDisk()
-		err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
-		if err != nil {
-			return err
+		if disk.AutoDelete {
+			err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil

pkg/compute/models/guestdisks.go

@@ -187,7 +187,7 @@ func (self *SGuestdisk) DoSave(ctx context.Context, driver string, cache string,
 func (self *SGuestdisk) GetDisk() *SDisk {
 	disk, err := DiskManager.FetchById(self.DiskId)
 	if err != nil {
-		log.Errorf("GetDisk fail: %s", err)
+		log.Errorf("GetDisk %s fail: %s", self.DiskId, err)
 		return nil
 	}
 	return disk.(*SDisk)
@@ -293,6 +293,10 @@ func (self *SGuestdisk) GetDetailedInfo() api.GuestDiskInfo {
 
 func (self *SGuestdisk) GetDetailedString() string {
 	disk := self.GetDisk()
+	if disk == nil {
+		return ""
+	}
+
 	var fs string
 	if len(disk.GetTemplateId()) > 0 {
 		fs = "root"

pkg/compute/models/guestdrivers.go

@@ -142,6 +142,8 @@ type IGuestDriver interface {
 	IsRebuildRootSupportChangeImage() bool
 	IsRebuildRootSupportChangeUEFI() bool
 
+	IsSupportdDcryptPasswordFromSecretKey() bool
+
 	RequestDeleteDetachedDisk(ctx context.Context, disk *SDisk, task taskman.ITask, isPurge bool) error
 	StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error
 	StartGuestAttachDiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error

pkg/compute/models/guests.go

@@ -2016,8 +2016,10 @@ func (self *SGuest) getNetworksDetails() string {
 func (self *SGuest) getDisksDetails() string {
 	var buf bytes.Buffer
 	for _, disk := range self.GetDisks() {
-		buf.WriteString(disk.GetDetailedString())
-		buf.WriteString("\n")
+		if details := disk.GetDetailedString(); len(details) > 0 {
+			buf.WriteString(details)
+			buf.WriteString("\n")
+		}
 	}
 	return buf.String()
 }
@@ -2537,7 +2539,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		guest.BillingType = extVM.GetBillingType()
-		guest.ExpiredAt = extVM.GetExpiredAt()
+		if expired := extVM.GetExpiredAt(); !expired.IsZero() {
+			guest.ExpiredAt = expired
+		}
 		if guest.GetDriver().IsSupportSetAutoRenew() {
 			guest.AutoRenew = extVM.IsAutoRenew()
 		}
@@ -2979,8 +2983,8 @@ func (self *SGuest) attach2Disk(ctx context.Context, disk *SDisk, userCred mccli
 	guestdisk.DiskId = disk.Id
 	guestdisk.GuestId = self.Id
 
-	defer lockman.ReleaseObject(ctx, self)
 	lockman.LockObject(ctx, self)
+	defer lockman.ReleaseObject(ctx, self)
 
 	guestdisk.Index = self.getDiskIndex()
 	err = guestdisk.DoSave(ctx, driver, cache, mountpoint)
@@ -4234,6 +4238,14 @@ func (self *SGuest) GetShortDesc(ctx context.Context) *jsonutils.JSONDict {
 		billingInfo.SCloudProviderInfo = host.getCloudProviderInfo()
 	}
 
+	if len(self.BackupHostId) > 0 {
+		backupHost := HostManager.FetchHostById(self.BackupHostId)
+		if backupHost != nil {
+			desc.Set("backup_host", jsonutils.NewString(backupHost.Name))
+			desc.Set("backup_host_id", jsonutils.NewString(backupHost.Id))
+		}
+	}
+
 	if priceKey := self.GetMetadata("ext:price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}
@@ -4257,40 +4269,49 @@ func (self *SGuest) saveOsType(userCred mcclient.TokenCredential, osType string)
 	return err
 }
 
+type sDeployInfo struct {
+	Os       string
+	Account  string
+	Key      string
+	Distro   string
+	Version  string
+	Arch     string
+	Language string
+}
+
 func (self *SGuest) SaveDeployInfo(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) {
+	deployInfo := sDeployInfo{}
+	data.Unmarshal(&deployInfo)
 	info := make(map[string]interface{})
-	if data.Contains("os") {
-		osName, _ := data.GetString("os")
-		self.saveOsType(userCred, osName)
-		info["os_name"] = osName
+	if len(deployInfo.Os) > 0 {
+		self.saveOsType(userCred, deployInfo.Os)
+		info["os_name"] = deployInfo.Os
 	}
-	if data.Contains("account") {
-		account, _ := data.GetString("account")
-		info["login_account"] = account
-		if data.Contains("key") {
-			key, _ := data.GetString("key")
-			info["login_key"] = key
+	driver := self.GetDriver()
+	if len(deployInfo.Account) > 0 {
+		info["login_account"] = deployInfo.Account
+		if len(deployInfo.Key) > 0 {
+			info["login_key"] = deployInfo.Key
+			if len(self.KeypairId) > 0 && !driver.IsSupportdDcryptPasswordFromSecretKey() { // Tencent Cloud does not support simultaneous setting of secret keys and passwords
+				info["login_key"], _ = seclib2.EncryptBase64(self.GetKeypairPublicKey(), "")
+			}
 			info["login_key_timestamp"] = timeutils.UtcNow()
 		} else {
 			info["login_key"] = "none"
 			info["login_key_timestamp"] = "none"
 		}
 	}
-	if data.Contains("distro") {
-		dist, _ := data.GetString("distro")
-		info["os_distribution"] = dist
+	if len(deployInfo.Distro) > 0 {
+		info["os_distribution"] = deployInfo.Distro
 	}
-	if data.Contains("version") {
-		ver, _ := data.GetString("version")
-		info["os_version"] = ver
+	if len(deployInfo.Version) > 0 {
+		info["os_version"] = deployInfo.Version
 	}
-	if data.Contains("arch") {
-		arch, _ := data.GetString("arch")
-		info["os_arch"] = arch
+	if len(deployInfo.Arch) > 0 {
+		info["os_arch"] = deployInfo.Arch
 	}
-	if data.Contains("language") {
-		lang, _ := data.GetString("language")
-		info["os_language"] = lang
+	if len(deployInfo.Language) > 0 {
+		info["os_language"] = deployInfo.Language
 	}
 	self.SetAllMetadata(ctx, info, userCred)
 	self.saveOldPassword(ctx, userCred)

pkg/compute/models/hosts.go

@@ -2519,11 +2519,7 @@ func (self *SHost) GetIHostAndProvider() (cloudprovider.ICloudHost, cloudprovide
 	}
 	ihost, err := iregion.GetIHostById(self.ExternalId)
 	if err != nil {
-		if err == cloudprovider.ErrNotFound {
-			return nil, nil, cloudprovider.ErrNotFound
-		}
-		log.Errorf("fail to find ihost by id %s %s", self.ExternalId, err)
-		return nil, nil, fmt.Errorf("fail to find ihost by id %s", err)
+		return nil, nil, errors.Wrapf(err, "iregion.GetIHostById(%s)", self.ExternalId)
 	}
 	return ihost, provider, nil
 }
@@ -4835,10 +4831,12 @@ func (manager *SHostManager) PingDetectionTask(ctx context.Context, userCred mcc
 		var host = new(SHost)
 		q.Row2Struct(rows, host)
 		host.SetModelManager(manager, host)
-		lockman.LockObject(ctx, host)
-		host.PerformOffline(ctx, userCred, nil, data)
-		host.MarkGuestUnknown(userCred)
-		lockman.ReleaseObject(ctx, host)
+		func() {
+			lockman.LockObject(ctx, host)
+			defer lockman.ReleaseObject(ctx, host)
+			host.PerformOffline(ctx, userCred, nil, data)
+			host.MarkGuestUnknown(userCred)
+		}()
 	}
 }
 
@@ -5450,3 +5448,15 @@ func (manager *SHostManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SHostManager) FetchHostByExtId(extid string) *SHost {
+	host := SHost{}
+	host.SetModelManager(manager, &host)
+	err := manager.Query().Equals("external_id", extid).First(&host)
+	if err != nil {
+		log.Errorf("fetchHostByExtId fail %s", err)
+		return nil
+	} else {
+		return &host
+	}
+}

pkg/compute/models/hostschedtags.go

@@ -44,7 +44,6 @@ func init() {
 				"schedtaghost",
 				"schedtaghosts",
 				HostManager,
-				SchedtagManager,
 			),
 		}
 		HostschedtagManager.SetVirtualObject(HostschedtagManager)
@@ -57,7 +56,7 @@ type SHostschedtag struct {
 	HostId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
 }
 
-func (manager *SHostschedtagManager) GetSlaveFieldName() string {
+func (manager *SHostschedtagManager) GetMasterFieldName() string {
 	return "host_id"
 }
 

pkg/compute/models/initdb.go

@@ -52,6 +52,7 @@ func InitDB() error {
 		LoadbalancerListenerRuleManager,
 		LoadbalancerBackendGroupManager,
 		LoadbalancerBackendManager,
+		AwsCachedLbbgManager,
 		LoadbalancerClusterManager,
 		SchedtagManager,
 		DynamicschedtagManager,
@@ -60,6 +61,12 @@ func InitDB() error {
 
 		ScheduledTaskActivityManager,
 		ExternalProjectManager,
+		CachedimageManager,
+		StoragecachedimageManager,
+		NetworkinterfacenetworkManager,
+		DBInstanceNetworkManager,
+		DBInstanceAccountManager,
+		DBInstanceDatabaseManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/loadbalancerawscachedlbb.go

@@ -159,16 +159,14 @@ func (man *SAwsCachedLbManager) SyncLoadbalancerBackends(ctx context.Context, us
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
 	for i := 0; i < len(added); i++ {
-		local, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
+		_, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/loadbalancerawscachedlbbg.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strconv"
 	"strings"
@@ -249,7 +250,6 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localLbgs = append(localLbgs, commondb[i])
 			remoteLbbgs = append(remoteLbbgs, commonext[i])
 			syncResult.Update()
@@ -277,7 +277,6 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
 			localLbgs = append(localLbgs, *new)
 			remoteLbbgs = append(remoteLbbgs, added[i])
 			syncResult.Add()
@@ -434,3 +433,30 @@ func (man *SAwsCachedLbbgManager) newFromCloudLoadbalancerBackendgroup(ctx conte
 	db.OpsLog.LogEvent(lbbg, db.ACT_CREATE, lbbg.GetShortDesc(ctx), userCred)
 	return lbbg, nil
 }
+
+func (man *SAwsCachedLbbgManager) InitializeData() error {
+	ret := []db.SMetadata{}
+	q := db.Metadata.Query().Equals("obj_type", man.Keyword())
+	err := db.FetchModelObjects(db.Metadata, q, &ret)
+	if err != nil {
+		if errors.Cause(err) != sql.ErrNoRows {
+			log.Debugf("SAwsCachedLbbgManager.InitializeData %s", err)
+		}
+
+		return nil
+	}
+
+	for i := range ret {
+		item := ret[i]
+		_, err := db.Update(&item, func() error {
+			return item.MarkDelete()
+		})
+		if err != nil {
+			log.Debugf("SAwsCachedLbbgManager.MarkDelete %s", err)
+			return nil
+		}
+	}
+
+	log.Debugf("SAwsCachedLbbgManager cleaned %d dirty data.", len(ret))
+	return nil
+}

pkg/compute/models/loadbalancerhuaweicachedlbb.go

@@ -162,16 +162,14 @@ func (man *SHuaweiCachedLbManager) SyncLoadbalancerBackends(ctx context.Context,
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
 	for i := 0; i < len(added); i++ {
-		local, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
+		_, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/loadbalancerhuaweicachedlbbg.go

@@ -222,7 +222,6 @@ func (man *SHuaweiCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.C
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localLbgs = append(localLbgs, commondb[i])
 			remoteLbbgs = append(remoteLbbgs, commonext[i])
 			syncResult.Update()
@@ -233,7 +232,6 @@ func (man *SHuaweiCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.C
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
 			localLbgs = append(localLbgs, *new)
 			remoteLbbgs = append(remoteLbbgs, added[i])
 			syncResult.Add()

pkg/compute/models/loadbalancerqcloudcachedlbb.go

@@ -234,16 +234,14 @@ func (man *SQcloudCachedLbManager) SyncLoadbalancerBackends(ctx context.Context,
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
 	for i := 0; i < len(added); i++ {
-		local, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
+		_, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}

pkg/compute/models/loadbalancerqcloudcachedlbbg.go

@@ -294,7 +294,6 @@ func (man *SQcloudCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.C
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localLbgs = append(localLbgs, commondb[i])
 			remoteLbbgs = append(remoteLbbgs, commonext[i])
 			syncResult.Update()
@@ -305,7 +304,6 @@ func (man *SQcloudCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.C
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, newlbbg, added[i])
 			localLbgs = append(localLbgs, *newlbbg)
 			remoteLbbgs = append(remoteLbbgs, added[i])
 			syncResult.Add()

pkg/compute/models/loadbalancerresource.go

@@ -75,6 +75,12 @@ func (self *SLoadbalancerResourceBase) GetCloudprovider() *SCloudprovider {
 	if vpc != nil {
 		return vpc.GetCloudprovider()
 	}
+
+	lb := self.GetLoadbalancer()
+	if lb != nil {
+		return lb.GetCloudprovider()
+	}
+
 	return nil
 }
 
@@ -161,23 +167,28 @@ func (manager *SLoadbalancerResourceBaseManager) FetchCustomizeColumns(
 
 	vpcList := make([]interface{}, len(rows))
 	zoneList := make([]interface{}, len(rows))
+	manList := make([]interface{}, len(rows))
 	for i := range rows {
 		rows[i] = api.LoadbalancerResourceInfo{}
 		if lb, ok := lbs[lbIds[i]]; ok {
 			rows[i].Loadbalancer = lb.Name
 			rows[i].VpcId = lb.VpcId
 			rows[i].ZoneId = lb.ZoneId
+			rows[i].ManagerId = lb.ManagerId
 		}
 		vpcList[i] = &SVpcResourceBase{rows[i].VpcId}
 		zoneList[i] = &SZoneResourceBase{rows[i].ZoneId}
+		manList[i] = &SManagedResourceBase{rows[i].ManagerId}
 	}
 
 	vpcRows := manager.SVpcResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, vpcList, fields, isList)
 	zoneRows := manager.SZoneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, zoneList, fields, isList)
+	manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, manList, fields, isList)
 
 	for i := range rows {
 		rows[i].VpcResourceInfo = vpcRows[i]
-		rows[i].ZoneResourceInfoBase = zoneRows[i].ZoneResourceInfoBase
+		rows[i].ZoneResourceInfo = zoneRows[i]
+		rows[i].ManagedResourceInfo = manRows[i]
 	}
 	return rows
 }

pkg/compute/models/loadbalancers.go

@@ -436,10 +436,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 		ChargeType:       lb.ChargeType,
 		LoadbalancerSpec: lb.LoadbalancerSpec,
 	}
-	iRegion, err := lb.GetIRegion()
-	if err != nil {
-		return nil, err
-	}
+
 	if len(lb.ZoneId) > 0 {
 		zone := lb.GetZone()
 		if zone == nil {
@@ -447,7 +444,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 		}
 		iZone, err := iRegion.GetIZoneById(zone.ExternalId)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "GetIZoneById")
 		}
 		params.ZoneID = iZone.GetId()
 	}
@@ -462,7 +459,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 		}
 		iVpc, err := iRegion.GetIVpcById(vpc.ExternalId)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "GetIVpcById")
 		}
 		params.VpcID = iVpc.GetId()
 	}
@@ -476,7 +473,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 		for i := range networks {
 			iNetwork, err := networks[i].GetINetwork()
 			if err != nil {
-				return nil, err
+				return nil, errors.Wrap(err, "GetINetwork")
 			}
 			params.NetworkIDs = append(params.NetworkIDs, iNetwork.GetId())
 		}

pkg/compute/models/managedresource.go

@@ -518,7 +518,7 @@ func _managedResourceFilterByAccount(managerIdFieldName string, q *sqlchemy.SQue
 	if len(cloudaccountArr) > 0 {
 		cpq := CloudaccountManager.Query().SubQuery()
 		subcpq := cpq.Query(cpq.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(cpq.Field("id"), cloudaccountArr),
+			sqlchemy.In(cpq.Field("id"), stringutils2.RemoveUtf8Strings(cloudaccountArr)),
 			sqlchemy.In(cpq.Field("name"), cloudaccountArr),
 		)).SubQuery()
 		subq := CloudproviderManager.Query("id").In("cloudaccount_id", subcpq).SubQuery()

pkg/compute/models/natgateways.go

@@ -363,7 +363,9 @@ func (self *SNatGateway) SyncWithCloudNatGateway(ctx context.Context, userCred m
 		factory, _ := provider.GetProviderFactory()
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extNat.GetBillingType()
-			self.ExpiredAt = extNat.GetExpiredAt()
+			if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extNat.IsAutoRenew()
 		}
 
@@ -407,7 +409,9 @@ func (manager *SNatGatewayManager) newFromCloudNatGateway(ctx context.Context, u
 	factory, _ := provider.GetProviderFactory()
 	if factory.IsSupportPrepaidResources() {
 		nat.BillingType = extNat.GetBillingType()
-		nat.ExpiredAt = extNat.GetExpiredAt()
+		if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+			nat.ExpiredAt = expired
+		}
 		nat.AutoRenew = extNat.IsAutoRenew()
 	}
 

pkg/compute/models/networkinterfacenetwork.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/netutils"
@@ -236,3 +237,23 @@ func (self *SNetworkinterfacenetwork) GetDetailJson() (jsonutils.JSONObject, err
 		"network":             network.Name,
 	}), nil
 }
+
+func (manager *SNetworkinterfacenetworkManager) InitializeData() error {
+	sq := NetworkInterfaceManager.Query("id")
+	q := manager.Query().NotIn("networkinterface_id", sq.SubQuery())
+	networks := []SNetworkinterfacenetwork{}
+	err := db.FetchModelObjects(manager, q, &networks)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range networks {
+		_, err = db.Update(&networks[i], func() error {
+			return networks[i].MarkDelete()
+		})
+		if err != nil {
+			return errors.Wrapf(err, "Delete %d", networks[i].RowId)
+		}
+	}
+	log.Debugf("SNetworkinterfacenetworkManager cleaned %d deprecated networkinterface ipAddrs.", len(networks))
+	return nil
+}

pkg/compute/models/networkinterfaces.go

@@ -286,8 +286,21 @@ func (self *SNetworkInterface) syncRemoveCloudNetworkInterface(ctx context.Conte
 
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return self.SetStatus(userCred, api.NETWORK_INTERFACE_STATUS_UNKNOWN, "sync to delete")
+		self.SetStatus(userCred, api.NETWORK_INTERFACE_STATUS_UNKNOWN, "sync to delete")
+		return errors.Wrapf(err, "ValidateDeleteCondition")
 	}
+
+	networks, err := self.GetNetworks()
+	if err != nil {
+		return errors.Wrapf(err, "GetNetworks")
+	}
+	for i := range networks {
+		err = networks[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "Delete networkinterfacenetwork %d", networks[i].RowId)
+		}
+	}
+
 	return self.Delete(ctx, userCred)
 }
 

pkg/compute/models/networks.go

@@ -155,17 +155,18 @@ func (self *SNetwork) ValidateDeleteCondition(ctx context.Context) error {
 	if err != nil {
 		return httperrors.NewInternalServerError("GetAllocatedNicCount fail %s", err)
 	}
-	if cnt > 0 {
-		return httperrors.NewNotEmptyError("not an empty network")
+	if cnt.Total > 0 {
+		return httperrors.NewNotEmptyError("not an empty network %s", jsonutils.Marshal(cnt).String())
 	}
 	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
 func (self *SNetwork) GetTotalNicCount() (int, error) {
-	total, err := self.GetAllocatedNicCount()
+	count, err := self.GetAllocatedNicCount()
 	if err != nil {
 		return -1, err
 	}
+	total := count.Total
 	cnt, err := self.GetReservedNicsCount()
 	if err != nil {
 		return -1, err
@@ -179,39 +180,50 @@ func (self *SNetwork) GetTotalNicCount() (int, error) {
 	return total, nil
 }
 
-func (self *SNetwork) GetAllocatedNicCount() (int, error) {
-	total := 0
-	cnt, err := self.GetGuestnicsCount()
+type sNicCount struct {
+	GuestNicCount        int
+	GroupNicCount        int
+	BaremetalNicCount    int
+	LoadbalancerNicCount int
+	DBInstanceNicCount   int
+	EipNicCount          int
+	Total                int
+}
+
+func (self *SNetwork) GetAllocatedNicCount() (*sNicCount, error) {
+	cnt := &sNicCount{Total: 0}
+	var err error
+	cnt.GuestNicCount, err = self.GetGuestnicsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetGuestnicsCount")
 	}
-	total += cnt
-	cnt, err = self.GetGroupNicsCount()
+	cnt.Total += cnt.GuestNicCount
+	cnt.GroupNicCount, err = self.GetGroupNicsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetGroupNicsCount")
 	}
-	total += cnt
-	cnt, err = self.GetBaremetalNicsCount()
+	cnt.Total += cnt.GroupNicCount
+	cnt.BaremetalNicCount, err = self.GetBaremetalNicsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetBaremetalNicsCount")
 	}
-	total += cnt
-	cnt, err = self.GetLoadbalancerIpsCount()
+	cnt.Total += cnt.BaremetalNicCount
+	cnt.LoadbalancerNicCount, err = self.GetLoadbalancerIpsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetLoadbalancerIpsCount")
 	}
-	total += cnt
-	cnt, err = self.GetDBInstanceIpsCount()
+	cnt.Total += cnt.LoadbalancerNicCount
+	cnt.DBInstanceNicCount, err = self.GetDBInstanceIpsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetDBInstanceIpsCount")
 	}
-	total += cnt
-	cnt, err = self.GetEipsCount()
+	cnt.Total += cnt.DBInstanceNicCount
+	cnt.EipNicCount, err = self.GetEipsCount()
 	if err != nil {
-		return -1, err
+		return nil, errors.Wrapf(err, "GetEipsCount")
 	}
-	total += cnt
-	return total, nil
+	cnt.Total += cnt.EipNicCount
+	return cnt, nil
 }
 
 /*验证elb network可用，并返回关联的region, zone,vpc, wire*/
@@ -294,6 +306,16 @@ func (self *SNetwork) GetLoadbalancerIpsCount() (int, error) {
 	return LoadbalancernetworkManager.Query().Equals("network_id", self.Id).CountWithError()
 }
 
+func (self *SNetwork) GetDBInstanceNetworks() ([]SDBInstanceNetwork, error) {
+	q := DBInstanceNetworkManager.Query().Equals("network_id", self.Id)
+	networks := []SDBInstanceNetwork{}
+	err := db.FetchModelObjects(DBInstanceNetworkManager, q, &networks)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return networks, nil
+}
+
 func (self *SNetwork) GetDBInstanceIpsCount() (int, error) {
 	return DBInstanceNetworkManager.Query().Equals("network_id", self.Id).CountWithError()
 }
@@ -477,7 +499,7 @@ func (self *SNetwork) GetNetAddr() netutils.IPV4Addr {
 }
 
 func (self *SNetwork) GetDNS() string {
-	if len(self.GuestDns) > 0 && len(self.GuestDomain) > 0 {
+	if len(self.GuestDns) > 0 {
 		return self.GuestDns
 	} else {
 		return options.Options.DNSServer
@@ -485,7 +507,7 @@ func (self *SNetwork) GetDNS() string {
 }
 
 func (self *SNetwork) GetDomain() string {
-	if len(self.GuestDns) > 0 && len(self.GuestDomain) > 0 {
+	if len(self.GuestDomain) > 0 {
 		return self.GuestDomain
 	} else {
 		return options.Options.DNSDomain
@@ -1701,6 +1723,7 @@ func (self *SNetwork) CustomizeDelete(ctx context.Context, userCred mcclient.Tok
 }
 
 func (self *SNetwork) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	DeleteResourceJointSchedtags(self, ctx, userCred)
 	db.OpsLog.LogEvent(self, db.ACT_DELOCATE, self.GetShortDesc(ctx), userCred)
 	self.SetStatus(userCred, api.NETWORK_STATUS_DELETED, "real delete")
 	self.ClearSchedDescCache()

pkg/compute/models/networkschedtags.go

@@ -44,7 +44,6 @@ func init() {
 				"schedtagnetwork",
 				"schedtagnetworks",
 				NetworkManager,
-				SchedtagManager,
 			),
 		}
 		NetworkschedtagManager.SetVirtualObject(NetworkschedtagManager)
@@ -57,7 +56,7 @@ type SNetworkschedtag struct {
 	NetworkId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
 }
 
-func (manager *SNetworkschedtagManager) GetSlaveFieldName() string {
+func (manager *SNetworkschedtagManager) GetMasterFieldName() string {
 	return "network_id"
 }
 

pkg/compute/models/purge.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 
 	"yunion.io/x/log"
@@ -652,7 +653,7 @@ func (manager *SSnapshotPolicyCacheManager) purgeAll(ctx context.Context, userCr
 
 func (spc *SSnapshotPolicyCache) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, spc)
-	defer lockman.LockObject(ctx, spc)
+	defer lockman.ReleaseObject(ctx, spc)
 	err := spc.ValidateDeleteCondition(ctx)
 	if err != nil {
 		return err
@@ -881,6 +882,20 @@ func (nic *SNetworkInterface) purge(ctx context.Context, userCred mcclient.Token
 	return nic.Delete(ctx, userCred)
 }
 
+func (net *SNetwork) purgeDBInstanceNetworks(ctx context.Context, userCred mcclient.TokenCredential) error {
+	dbNets, err := net.GetDBInstanceNetworks()
+	if err != nil {
+		return errors.Wrapf(err, "GetDBInstanceNetworks")
+	}
+	for i := range dbNets {
+		err = dbNets[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "Delete %d", dbNets[i].RowId)
+		}
+	}
+	return nil
+}
+
 func (net *SNetwork) purgeNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCredential) error {
 	networkinterfaceIds := NetworkinterfacenetworkManager.Query("networkinterface_id").Equals("network_id", net.Id).Distinct().SubQuery()
 	q := NetworkInterfaceManager.Query().In("id", networkinterfaceIds)
@@ -904,37 +919,42 @@ func (net *SNetwork) purge(ctx context.Context, userCred mcclient.TokenCredentia
 
 	err := net.purgeGuestnetworks(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeGuestnetworks")
 	}
 	err = net.purgeHostnetworks(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeHostnetworks")
 	}
 	err = net.purgeGroupnetworks(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeGroupnetworks")
 	}
 	err = net.purgeLoadbalancernetworks(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeLoadbalancernetworks")
 	}
 	err = net.purgeEipnetworks(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeEipnetworks")
 	}
 	err = net.purgeReservedIps(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeReservedIps")
 	}
 
 	err = net.purgeNetworkInterfaces(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeNetworkInterfaces")
+	}
+
+	err = net.purgeDBInstanceNetworks(ctx, userCred)
+	if err != nil {
+		return errors.Wrapf(err, "purgeDBInstanceNetworks")
 	}
 
 	err = net.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "ValidateDeleteCondition")
 	}
 	return net.RealDelete(ctx, userCred)
 }
@@ -1334,13 +1354,16 @@ func (network *SDBInstanceNetwork) purge(ctx context.Context, userCred mcclient.
 
 	err := network.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "ValidateDeleteCondition")
 	}
 	return network.Delete(ctx, userCred)
 }
 
 func (instance *SDBInstance) purgeNetwork(ctx context.Context, userCred mcclient.TokenCredential) error {
-	network, _ := instance.GetDBNetwork()
+	network, err := instance.GetDBNetwork()
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		return errors.Wrapf(err, "GetDBNetwork")
+	}
 	if network != nil {
 		return network.purge(ctx, userCred)
 	}
@@ -1369,22 +1392,22 @@ func (instance *SDBInstance) Purge(ctx context.Context, userCred mcclient.TokenC
 
 	err := instance.purgeAccounts(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeAccounts")
 	}
 
 	err = instance.purgeDatabases(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeDatabases")
 	}
 
 	err = instance.purgeParameters(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeParameters")
 	}
 
 	err = instance.purgeNetwork(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "purgeNetwork")
 	}
 
 	err = instance.PurgeBackups(ctx, userCred, api.BACKUP_MODE_AUTOMATED)
@@ -1394,7 +1417,7 @@ func (instance *SDBInstance) Purge(ctx context.Context, userCred mcclient.TokenC
 
 	err = instance.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "ValidateDeleteCondition")
 	}
 
 	return instance.RealDelete(ctx, userCred)

pkg/compute/models/regiondrivers.go

@@ -16,7 +16,6 @@ package models
 
 import (
 	"context"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -155,7 +154,6 @@ type IDBInstanceDriver interface {
 
 	RequestCreateDBInstance(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *SDBInstance, task taskman.ITask) error
 	RequestCreateDBInstanceBackup(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, backup *SDBInstanceBackup, task taskman.ITask) error
-	RequestRenewDBInstance(instance *SDBInstance, bc billing.SBillingCycle) (time.Time, error)
 	RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, task taskman.ITask) error
 
 	IsSupportedDBInstance() bool

pkg/compute/models/regionquota.go

@@ -196,7 +196,8 @@ func (self *SRegionQuota) FetchUsage(ctx context.Context) error {
 	self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000)
 	self.ObjectCnt = bucketUsage.Objects
 
-	self.Rds, _ = DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	rdsUsage, _ := DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	self.Rds = rdsUsage.TotalRdsCount
 	self.Cache, _ = ElasticcacheManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
 
 	self.Loadbalancer, _ = LoadbalancerManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)

pkg/compute/models/scalinggroup_guest.go

@@ -55,6 +55,10 @@ type SScalingGroupGuest struct {
 	Manual         tristate.TriState `nullable:"false" default:"false"`
 }
 
+func (sggm *SScalingGroupGuestManager) GetSlaveFieldName() string {
+	return "scaling_group_id"
+}
+
 func (sggm *SScalingGroupGuestManager) Attach(ctx context.Context, scaligGroupId, guestId string, manual bool) error {
 	sgg := &SScalingGroupGuest{
 		SGuestJointsBase: SGuestJointsBase{

pkg/compute/models/schedtagjoint.go

@@ -44,7 +44,6 @@ func NewSchedtagJointsManager(
 	keyword string,
 	keywordPlural string,
 	master db.IStandaloneModelManager,
-	slave db.IStandaloneModelManager,
 ) *SSchedtagJointsManager {
 	return &SSchedtagJointsManager{
 		SJointResourceBaseManager: db.NewJointResourceBaseManager(
@@ -53,7 +52,7 @@ func NewSchedtagJointsManager(
 			keyword,
 			keywordPlural,
 			master,
-			slave,
+			SchedtagManager,
 		),
 	}
 }
@@ -64,7 +63,7 @@ type SSchedtagJointsBase struct {
 	SchedtagId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // =Column(VARCHAR(36, charset='ascii'), nullable=False)
 }
 
-func (manager *SSchedtagJointsManager) GetMasterFieldName() string {
+func (manager *SSchedtagJointsManager) GetSlaveFieldName() string {
 	return "schedtag_id"
 }
 
@@ -118,7 +117,7 @@ func (man *SSchedtagJointsManager) AllowListDescendent(ctx context.Context, user
 	return db.IsAdminAllowList(userCred, man)
 }
 
-func (man *SSchedtagJointsManager) GetMasterIdKey(m db.IJointModelManager) string {
+func (man *SSchedtagJointsManager) GetResourceIdKey(m db.IJointModelManager) string {
 	return fmt.Sprintf("%s_id", m.GetMasterManager().Keyword())
 }
 

pkg/compute/models/schedtags.go

@@ -51,7 +51,7 @@ var STRATEGY_LIST = api.STRATEGY_LIST
 
 type ISchedtagJointManager interface {
 	db.IJointModelManager
-	GetMasterIdKey(db.IJointModelManager) string
+	GetResourceIdKey(db.IJointModelManager) string
 }
 
 type ISchedtagJointModel interface {
@@ -367,9 +367,9 @@ func (self *SSchedtag) GetObjectQuery() *sqlchemy.SQuery {
 	objs := masterMan.Query().SubQuery()
 	objschedtags := jointMan.Query().SubQuery()
 	q := objs.Query()
-	q = q.Join(objschedtags, sqlchemy.AND(sqlchemy.Equals(objschedtags.Field(jointMan.GetMasterIdKey(jointMan)), objs.Field("id")),
+	q = q.Join(objschedtags, sqlchemy.AND(sqlchemy.Equals(objschedtags.Field(jointMan.GetResourceIdKey(jointMan)), objs.Field("id")),
 		sqlchemy.IsFalse(objschedtags.Field("deleted"))))
-	q = q.Filter(sqlchemy.IsTrue(objs.Field("enabled")))
+	// q = q.Filter(sqlchemy.IsTrue(objs.Field("enabled")))
 	q = q.Filter(sqlchemy.Equals(objschedtags.Field("schedtag_id"), self.Id))
 	return q
 }
@@ -379,7 +379,8 @@ func (self *SSchedtag) GetJointManager() ISchedtagJointManager {
 }
 
 func (self *SSchedtag) GetObjectCount() (int, error) {
-	return self.GetJointManager().Query().Equals("schedtag_id", self.Id).CountWithError()
+	q := self.GetObjectQuery()
+	return q.CountWithError()
 }
 
 func (self *SSchedtag) getSchedPoliciesCount() (int, error) {
@@ -468,7 +469,7 @@ func (self *SSchedtag) GetShortDescV2(ctx context.Context) api.SchedtagShortDesc
 
 func GetResourceJointSchedtags(obj IModelWithSchedtag) ([]ISchedtagJointModel, error) {
 	jointMan := obj.GetSchedtagJointManager()
-	q := jointMan.Query().Equals(jointMan.GetMasterIdKey(jointMan), obj.GetId())
+	q := jointMan.Query().Equals(jointMan.GetResourceIdKey(jointMan), obj.GetId())
 	jointTags := make([]ISchedtagJointModel, 0)
 	rows, err := q.Rows()
 	if err != nil {
@@ -500,7 +501,7 @@ func GetSchedtags(jointMan ISchedtagJointManager, masterId string) []SSchedtag {
 	q := schedtags.Query()
 	q = q.Join(objschedtags, sqlchemy.AND(sqlchemy.Equals(objschedtags.Field("schedtag_id"), schedtags.Field("id")),
 		sqlchemy.IsFalse(objschedtags.Field("deleted"))))
-	q = q.Filter(sqlchemy.Equals(objschedtags.Field(jointMan.GetMasterIdKey(jointMan)), masterId))
+	q = q.Filter(sqlchemy.Equals(objschedtags.Field(jointMan.GetResourceIdKey(jointMan)), masterId))
 	err := db.FetchModelObjects(SchedtagManager, q, &tags)
 	if err != nil {
 		log.Errorf("GetSchedtags error: %s", err)
@@ -552,7 +553,7 @@ func PerformSetResourceSchedtag(obj IModelWithSchedtag, ctx context.Context, use
 			if newTagObj, err := db.NewModelObject(jointMan); err != nil {
 				return nil, httperrors.NewGeneralError(err)
 			} else {
-				objectKey := jointMan.GetMasterIdKey(jointMan)
+				objectKey := jointMan.GetResourceIdKey(jointMan)
 				createData := jsonutils.NewDict()
 				createData.Add(jsonutils.NewString(setTagId), "schedtag_id")
 				createData.Add(jsonutils.NewString(obj.GetId()), objectKey)

pkg/compute/models/secgroupcache.go

@@ -355,6 +355,39 @@ func (self *SSecurityGroupCache) GetSecgroup() (*SSecurityGroup, error) {
 	return model.(*SSecurityGroup), nil
 }
 
+func (self *SSecurityGroupCache) syncWithCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudSecurityGroup) error {
+	_, err := db.Update(self, func() error {
+		self.Status = api.SECGROUP_CACHE_STATUS_READY
+		self.Name = ext.GetName()
+		self.Description = ext.GetDescription()
+		self.ExternalProjectId = ext.GetProjectId()
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "db.Update")
+	}
+	secgroup, err := self.GetSecgroup()
+	if err != nil {
+		return errors.Wrapf(err, "GetSecurity")
+	}
+	cacheCount, err := secgroup.GetSecgroupCacheCount()
+	if err != nil {
+		return errors.Wrapf(err, "GetSecgroupCacheCount")
+	}
+	if cacheCount > 1 {
+		return nil
+	}
+	info, err := SecurityGroupManager.getRuleInfo(provider, ext)
+	if err != nil {
+		return errors.Wrapf(err, "getRuleInfo")
+	}
+	err = secgroup.SyncSecurityGroupRules(ctx, userCred, info)
+	if err != nil {
+		return errors.Wrapf(err, "SyncSecurityGroupRules")
+	}
+	return nil
+}
+
 func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
@@ -415,25 +448,18 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 	}
 
 	for i := 0; i < len(commondb); i++ {
-		_, err = db.Update(&commondb[i], func() error {
-			commondb[i].Status = api.SECGROUP_CACHE_STATUS_READY
-			commondb[i].Name = commonext[i].GetName()
-			commondb[i].Description = commonext[i].GetDescription()
-			commondb[i].ExternalProjectId = commonext[i].GetProjectId()
-			return nil
-		})
+		err = commondb[i].syncWithCloudSecurityGroup(ctx, userCred, provider, commonext[i])
 		if err != nil {
-			syncResult.UpdateError(err)
-		} else {
-			syncResult.Update()
+			syncResult.UpdateError(errors.Wrapf(err, "syncWithCloudSecurityGroup"))
+			continue
 		}
+		syncResult.Update()
 	}
 
-	//相同的不能同步, 原因: 多个平台的安全组可能共用一个本地安全组,下面仅仅是新加的安全组
 	for i := 0; i < len(added); i++ {
 		secgroup, err := SecurityGroupManager.newFromCloudSecgroup(ctx, userCred, provider, added[i])
 		if err != nil {
-			syncResult.AddError(err)
+			syncResult.AddError(errors.Wrapf(err, "newFromCloudSecgroup"))
 			continue
 		}
 		if secgroup.ProjectId != provider.ProjectId {
@@ -450,7 +476,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 		}
 		cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id, added[i].GetProjectId())
 		if err != nil {
-			syncResult.AddError(fmt.Errorf("failed to create secgroup cache for secgroup %s(%s) provider: %s: %s", secgroup.Name, secgroup.Name, provider.Name, err))
+			syncResult.AddError(errors.Wrapf(err, "NewCache for secgroup %s provider %s", secgroup.Name, provider.Name))
 			continue
 		}
 		_, err = db.Update(cache, func() error {
@@ -461,7 +487,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 			return nil
 		})
 		if err != nil {
-			syncResult.AddError(err)
+			syncResult.AddError(errors.Wrapf(err, "db.Update"))
 			continue
 		}
 		localSecgroups = append(localSecgroups, *secgroup)
@@ -560,3 +586,20 @@ func (manager *SSecurityGroupCacheManager) ListItemExportKeys(ctx context.Contex
 
 	return q, nil
 }
+
+func (self *SSecurityGroupCache) GetISecurityGroup() (cloudprovider.ICloudSecurityGroup, error) {
+	if len(self.ExternalId) == 0 {
+		return nil, errors.Wrapf(cloudprovider.ErrNotFound, "empty externalId")
+	}
+
+	manager := self.GetCloudprovider()
+	if manager == nil {
+		return nil, errors.Wrapf(cloudprovider.ErrNotFound, "failed to found manager")
+	}
+
+	iRegion, err := self.GetIRegion()
+	if err != nil {
+		return nil, errors.Wrapf(err, "GetIRegion")
+	}
+	return iRegion.GetISecurityGroupById(self.ExternalId)
+}

pkg/compute/models/secgrouprules.go

@@ -22,7 +22,6 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/util/stringutils"
@@ -462,8 +461,7 @@ func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSec
 	return rules, nil
 }
 
-func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules cloudprovider.SecurityRuleSet) compare.SyncResult {
-	syncResult := compare.SyncResult{}
+func (self *SSecurityGroup) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, rules cloudprovider.SecurityRuleSet) error {
 	priority, prePriority := 10, 0
 	for i := 0; i < len(rules); i++ {
 		// 这里避免了Rule规则优先级在 1-100之外的问题,ext.GetRules()不需要进行优先级转换
@@ -472,19 +470,17 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre
 		}
 		prePriority = rules[i].Priority
 		rules[i].Priority = priority
-		_, err := manager.newFromCloudSecurityGroup(ctx, userCred, rules[i], secgroup)
+		_, err := self.newFromCloudSecurityGroupRule(ctx, userCred, rules[i])
 		if err != nil {
-			syncResult.AddError(err)
-			continue
+			return errors.Wrapf(err, "newFromCloudSecurityGroupRule")
 		}
-		syncResult.Add()
 	}
-	return syncResult
+	return nil
 }
 
-func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) {
-	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
-	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
+func (self *SSecurityGroup) newFromCloudSecurityGroupRule(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule) (*SSecurityGroupRule, error) {
+	lockman.LockObject(ctx, self)
+	defer lockman.ReleaseObject(ctx, self)
 
 	protocol := rule.Protocol
 	if len(protocol) == 0 {
@@ -505,11 +501,11 @@ func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.
 		Action:      string(rule.Action),
 		Description: rule.Description,
 	}
-	secrule.SecgroupId = secgroup.Id
+	secrule.SecgroupId = self.Id
 
-	err := manager.TableSpec().Insert(ctx, secrule)
+	err := SecurityGroupRuleManager.TableSpec().Insert(ctx, secrule)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "SecurityGroupRuleManager.Insert")
 	}
 	return secrule, nil
 }

pkg/compute/models/secgroups.go

@@ -37,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
@@ -539,6 +540,18 @@ func totalSecurityGroupCount(scope rbacutils.TRbacScope, ownerId mcclient.IIdent
 	return q.CountWithError()
 }
 
+func (self *SSecurityGroup) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "purge")
+}
+
+func (self *SSecurityGroup) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	err := self.ValidateDeleteCondition(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return nil, self.StartDeleteSecurityGroupTask(ctx, userCred, true, "")
+}
+
 func (self *SSecurityGroup) AllowPerformUncacheSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "uncache-secgroup")
 }
@@ -821,7 +834,57 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err
 	}
 }
 
-func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) {
+func (self *SSecurityGroup) cleanRules(ctx context.Context, userCred mcclient.TokenCredential) error {
+	rules := []SSecurityGroupRule{}
+	q := SecurityGroupRuleManager.Query().Equals("secgroup_id", self.Id)
+	err := db.FetchModelObjects(SecurityGroupRuleManager, q, &rules)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range rules {
+		err = rules[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "DeleteRule(%s)", rules[i].Id)
+		}
+	}
+	return nil
+}
+
+func (self *SSecurityGroup) SyncSecurityGroupRules(ctx context.Context, userCred mcclient.TokenCredential, info *sRuleInfo) error {
+	inRules := cloudprovider.AddDefaultRule(info.inRules, info.defaultInRule, "in:deny any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules)
+	cloudprovider.SortSecurityRule(inRules, info.order, info.onlyAllowRules)
+	outRules := cloudprovider.AddDefaultRule(info.outRules, info.defaultOutRule, "out:allow any", info.order, info.minPriority, info.maxPriority, info.onlyAllowRules)
+	cloudprovider.SortSecurityRule(outRules, info.order, info.onlyAllowRules)
+
+	err := self.cleanRules(ctx, userCred)
+	if err != nil {
+		return errors.Wrapf(err, "cleanRules")
+	}
+
+	err = self.SyncRules(ctx, userCred, inRules)
+	if err != nil {
+		return errors.Wrapf(err, "SyncInRules")
+	}
+	err = self.SyncRules(ctx, userCred, outRules)
+	if err != nil {
+		return errors.Wrapf(err, "SyncOutRules")
+	}
+	return nil
+}
+
+type sRuleInfo struct {
+	rules          []cloudprovider.SecurityRule
+	inRules        []cloudprovider.SecurityRule
+	outRules       []cloudprovider.SecurityRule
+	defaultInRule  cloudprovider.SecurityRule
+	defaultOutRule cloudprovider.SecurityRule
+	order          cloudprovider.TPriorityOrder
+	onlyAllowRules bool
+	maxPriority    int
+	minPriority    int
+}
+
+func (manager *SSecurityGroupManager) getRuleInfo(provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*sRuleInfo, error) {
 	regionDriver, err := provider.GetRegionDriver()
 	if err != nil {
 		return nil, errors.Wrap(err, "provider.GetRegionDriver")
@@ -832,36 +895,48 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, errors.Wrap(err, "extSec.GetRules")
 	}
 
-	inRules := []cloudprovider.SecurityRule{}
-	outRules := []cloudprovider.SecurityRule{}
+	info := &sRuleInfo{
+		rules:          rules,
+		inRules:        []cloudprovider.SecurityRule{},
+		outRules:       []cloudprovider.SecurityRule{},
+		defaultInRule:  regionDriver.GetDefaultSecurityGroupInRule(),
+		defaultOutRule: regionDriver.GetDefaultSecurityGroupOutRule(),
+		order:          regionDriver.GetSecurityGroupRuleOrder(),
+		onlyAllowRules: regionDriver.IsOnlySupportAllowRules(),
+		maxPriority:    regionDriver.GetSecurityGroupRuleMaxPriority(),
+		minPriority:    regionDriver.GetSecurityGroupRuleMinPriority(),
+	}
+
 	for i := range rules {
 		if rules[i].Direction == secrules.DIR_IN {
-			inRules = append(inRules, rules[i])
+			info.inRules = append(info.inRules, rules[i])
 		} else {
-			outRules = append(outRules, rules[i])
+			info.outRules = append(info.outRules, rules[i])
 		}
 	}
+	return info, nil
+}
 
-	maxPriority := regionDriver.GetSecurityGroupRuleMaxPriority()
-	minPriority := regionDriver.GetSecurityGroupRuleMinPriority()
-
-	defaultInRule := regionDriver.GetDefaultSecurityGroupInRule()
-	defaultOutRule := regionDriver.GetDefaultSecurityGroupOutRule()
-	order := regionDriver.GetSecurityGroupRuleOrder()
-	onlyAllowRules := regionDriver.IsOnlySupportAllowRules()
-
-	// 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组
-	secgroups := []SSecurityGroup{}
-	q := manager.Query().Equals("domain_id", provider.DomainId)
-	err = db.FetchModelObjects(manager, q, &secgroups)
+func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) {
+	info, err := manager.getRuleInfo(provider, extSec)
 	if err != nil {
-		return nil, errors.Wrap(err, "db.FetchModelObjects")
+		return nil, errors.Wrapf(err, "getRuleInfo")
 	}
-	for i := range secgroups {
-		localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules(""))
-		_, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
-		if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 {
-			return &secgroups[i], nil
+
+	if options.Options.EnableAutoMergeSecurityGroup {
+		// 查询与provider在同域的安全组，比对寻找一个与云上安全组规则相同的安全组
+		secgroups := []SSecurityGroup{}
+		q := manager.Query().Equals("domain_id", provider.DomainId)
+		err = db.FetchModelObjects(manager, q, &secgroups)
+		if err != nil {
+			return nil, errors.Wrap(err, "db.FetchModelObjects")
+		}
+		for i := range secgroups {
+			localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules(""))
+			_, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(info.minPriority, info.maxPriority, info.order, localRules, info.rules, info.defaultInRule, info.defaultOutRule, info.onlyAllowRules, false)
+			if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 {
+				return &secgroups[i], nil
+			}
 		}
 	}
 
@@ -870,31 +945,27 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 
 	secgroup := SSecurityGroup{}
 	secgroup.SetModelManager(manager, &secgroup)
-	newName, err := db.GenerateName(manager, userCred, extSec.GetName())
+	secgroup.Name, err = db.GenerateName(manager, userCred, extSec.GetName())
 	if err != nil {
 		return nil, err
 	}
 
-	secgroup.Name = newName
+	secgroup.Status = api.SECGROUP_STATUS_READY
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId
 
-	if err := manager.TableSpec().Insert(ctx, &secgroup); err != nil {
-		return nil, err
+	err = manager.TableSpec().Insert(ctx, &secgroup)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Insert")
 	}
 
-	//这里必须先同步下规则,不然下次对比此安全组规则为空
-	inRules = cloudprovider.AddDefaultRule(inRules, defaultInRule, "in:deny any", order, minPriority, maxPriority, onlyAllowRules)
-	cloudprovider.SortSecurityRule(inRules, order, onlyAllowRules)
-	outRules = cloudprovider.AddDefaultRule(outRules, defaultOutRule, "out:allow any", order, minPriority, maxPriority, onlyAllowRules)
-	cloudprovider.SortSecurityRule(outRules, order, onlyAllowRules)
-
-	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, inRules)
-	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, outRules)
+	err = secgroup.SyncSecurityGroupRules(ctx, userCred, info)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SyncSecurityGroupRules")
+	}
 
 	db.OpsLog.LogEvent(&secgroup, db.ACT_CREATE, secgroup.GetShortDesc(ctx), userCred)
-
 	return &secgroup, nil
 }
 
@@ -942,12 +1013,11 @@ func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenC
 func (manager *SSecurityGroupManager) InitializeData() error {
 	_, err := manager.FetchById("default")
 	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("find default secgroup fail %s", err)
-		return err
+		return errors.Wrapf(err, `manager.FetchById("default")`)
 	}
-	if err == sql.ErrNoRows {
-		var secGrp *SSecurityGroup
-		secGrp = &SSecurityGroup{}
+	if errors.Cause(err) == sql.ErrNoRows {
+		log.Debugf("Init default secgroup")
+		secGrp := &SSecurityGroup{}
 		secGrp.SetModelManager(manager, secGrp)
 		secGrp.Id = "default"
 		secGrp.Name = "Default"
@@ -956,11 +1026,11 @@ func (manager *SSecurityGroupManager) InitializeData() error {
 		secGrp.DomainId = auth.AdminCredential().GetProjectDomainId()
 		// secGrp.IsEmulated = false
 		secGrp.IsPublic = true
+		secGrp.Deleted = false
 		secGrp.PublicScope = string(rbacutils.ScopeSystem)
-		err = manager.TableSpec().Insert(context.TODO(), secGrp)
+		err = manager.TableSpec().InsertOrUpdate(context.TODO(), secGrp)
 		if err != nil {
-			log.Errorf("Insert default secgroup failed!!! %s", err)
-			return err
+			return errors.Wrapf(err, "Insert default secgroup")
 		}
 
 		defRule := SSecurityGroupRule{}
@@ -973,8 +1043,7 @@ func (manager *SSecurityGroupManager) InitializeData() error {
 		defRule.SecgroupId = "default"
 		err = SecurityGroupRuleManager.TableSpec().Insert(context.TODO(), &defRule)
 		if err != nil {
-			log.Errorf("Insert default secgroup rule fail %s", err)
-			return err
+			return errors.Wrapf(err, "Insert default secgroup rule")
 		}
 	}
 	guests := make([]SGuest, 0)
@@ -1035,10 +1104,12 @@ func (self *SSecurityGroup) GetSecurityGroupCaches() []SSecurityGroupCache {
 }
 
 func (self *SSecurityGroup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	return self.StartDeleteSecurityGroupTask(ctx, userCred, jsonutils.NewDict(), "")
+	return self.StartDeleteSecurityGroupTask(ctx, userCred, false, "")
 }
 
-func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error {
+func (self *SSecurityGroup) StartDeleteSecurityGroupTask(ctx context.Context, userCred mcclient.TokenCredential, isPurge bool, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewBool(isPurge), "purge")
 	self.SetStatus(userCred, api.SECGROUP_STATUS_DELETING, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "SecurityGroupDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {

pkg/compute/models/snapshotpolicy.go

@@ -310,11 +310,11 @@ func (sp *SSnapshotPolicy) DetachAfterDelete(ctx context.Context, userCred mccli
 
 func (sp *SSnapshotPolicy) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	// check if sp bind to some disks
-	sds, err := SnapshotPolicyDiskManager.FetchAllBySnapshotpolicyID(ctx, userCred, sp.GetId())
+	count, err := SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	if err != nil {
-		return errors.Wrap(err, "fetch bind info failed")
+		return errors.Wrap(err, "unable to FetchDiskCountBySPID")
 	}
-	if len(sds) != 0 {
+	if count != 0 {
 		return httperrors.NewBadRequestError("Couldn't delete snapshot policy binding to disks")
 	}
 	sp.SetStatus(userCred, api.SNAPSHOT_POLICY_DELETING, "")

pkg/compute/models/snapshotpolicycache.go

@@ -484,7 +484,7 @@ func (spc *SSnapshotPolicyCache) DeleteCloudSnapshotPolicy() error {
 			return err
 		}
 		cloudSp, err := iregion.GetISnapshotPolicyById(spc.ExternalId)
-		if err == cloudprovider.ErrNotFound {
+		if errors.Cause(err) == cloudprovider.ErrNotFound {
 			return nil
 		}
 		if err != nil {

pkg/compute/models/storagecachedimages.go

@@ -331,7 +331,7 @@ func (self *SStoragecachedimage) markDeleting(ctx context.Context, userCred mccl
 	}
 
 	if !isForce && !utils.IsInStringArray(self.Status,
-		[]string{api.CACHED_IMAGE_STATUS_READY, api.CACHED_IMAGE_STATUS_DELETING, api.CACHED_IMAGE_STATUS_CACHE_FAILED}) {
+		[]string{api.CACHED_IMAGE_STATUS_ACTIVE, api.CACHED_IMAGE_STATUS_DELETING, api.CACHED_IMAGE_STATUS_CACHE_FAILED}) {
 		return httperrors.NewInvalidStatusError("Cannot uncache in status %s", self.Status)
 	}
 	_, err = db.Update(self, func() error {
@@ -439,17 +439,17 @@ func (self *SStoragecachedimage) syncRemoveCloudImage(ctx context.Context, userC
 	return nil
 }
 
-func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error {
+func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	cachedImage := self.GetCachedimage()
 	if len(cachedImage.ExternalId) > 0 {
 		self.SetStatus(userCred, image.GetStatus(), "")
-		return cachedImage.syncWithCloudImage(ctx, userCred, image)
+		return cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, managerId)
 	} else {
 		return nil
 	}
 }
 
-func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage, cache *SStoragecache) error {
+func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, cache *SStoragecache) error {
 	var cachedImage *SCachedimage
 	imgObj, err := db.FetchByExternalId(CachedimageManager, image.GetGlobalId())
 	if err != nil {
@@ -472,7 +472,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context
 		}
 		if cachedImage == nil {
 			// no such image
-			cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, image)
+			cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, ownerId, image, cache.ManagerId)
 			if err != nil {
 				log.Errorf("CachedimageManager.newFromCloudImage fail %s", err)
 				return err
@@ -482,7 +482,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context
 		cachedImage = imgObj.(*SCachedimage)
 	}
 	if len(cachedImage.ExternalId) > 0 {
-		cachedImage.syncWithCloudImage(ctx, userCred, image)
+		cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, cache.ManagerId)
 	}
 	scimg := manager.Register(ctx, userCred, cache.GetId(), cachedImage.GetId(), image.GetStatus())
 	if scimg == nil {
@@ -553,3 +553,22 @@ func (manager *SStoragecachedimageManager) OrderByExtraFields(
 
 	return q, nil
 }
+
+func (manager *SStoragecachedimageManager) InitializeData() error {
+	images := []SStoragecachedimage{}
+	q := manager.Query().Equals("status", "ready")
+	err := db.FetchModelObjects(manager, q, &images)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range images {
+		_, err := db.Update(&images[i], func() error {
+			images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE
+			return nil
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update(%d)", images[i].RowId)
+		}
+	}
+	return nil
+}

pkg/compute/models/storagecaches.go

@@ -610,6 +610,13 @@ func (cache *SStoragecache) SyncCloudImages(
 		return syncResult
 	}
 
+	var syncOwnerId mcclient.IIdentityProvider
+
+	provider := cache.GetCloudprovider()
+	if provider != nil {
+		syncOwnerId = provider.GetOwnerId()
+	}
+
 	removed := make([]SStoragecachedimage, 0)
 	commondb := make([]SStoragecachedimage, 0)
 	commonext := make([]cloudprovider.ICloudImage, 0)
@@ -631,7 +638,7 @@ func (cache *SStoragecache) SyncCloudImages(
 		}
 	}
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudImage(ctx, userCred, commonext[i])
+		err = commondb[i].syncWithCloudImage(ctx, userCred, syncOwnerId, commonext[i], cache.ManagerId)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -639,7 +646,7 @@ func (cache *SStoragecache) SyncCloudImages(
 		}
 	}
 	for i := 0; i < len(added); i += 1 {
-		err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, added[i], cache)
+		err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, syncOwnerId, added[i], cache)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
@@ -662,7 +669,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool {
 		log.Debugf("image %s is not a customized image, no need to cache", imageId)
 		return false
 	}
-	cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY})
+	cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE})
 	for i := range cachedImages {
 		if cachedImages[i].Id == imageId {
 			// already cached
@@ -675,7 +682,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool {
 }
 
 func (self *SStoragecache) StartRelinquishLeastUsedCachedImageTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, parentTaskId string) error {
-	cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY})
+	cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE})
 	leastUsedIdx := -1
 	leastRefCount := -1
 	for i := range cachedImages {

pkg/compute/models/storageschedtags.go

@@ -44,7 +44,6 @@ func init() {
 				"schedtagstorage",
 				"schedtagstorages",
 				StorageManager,
-				SchedtagManager,
 			),
 		}
 		StorageschedtagManager.SetVirtualObject(StorageschedtagManager)
@@ -57,7 +56,7 @@ type SStorageschedtag struct {
 	StorageId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
 }
 
-func (manager *SStorageschedtagManager) GetSlaveFieldName() string {
+func (manager *SStorageschedtagManager) GetMasterFieldName() string {
 	return "storage_id"
 }
 

pkg/compute/models/syncworkers.go

@@ -26,9 +26,10 @@ import (
 )
 
 var (
-	syncAccountWorker *appsrv.SWorkerManager
-	syncWorkers       []*appsrv.SWorkerManager
-	syncWorkerRing    *hashring.HashRing
+	syncSecgroupWorker *appsrv.SWorkerManager
+	syncAccountWorker  *appsrv.SWorkerManager
+	syncWorkers        []*appsrv.SWorkerManager
+	syncWorkerRing     *hashring.HashRing
 )
 
 func InitSyncWorkers(count int) {
@@ -50,6 +51,12 @@ func InitSyncWorkers(count int) {
 		2048,
 		true,
 	)
+	syncSecgroupWorker = appsrv.NewWorkerManager(
+		"syncSecgroupProbeWorkerManager",
+		1,
+		2048,
+		true,
+	)
 }
 
 func RunSyncCloudproviderRegionTask(key string, syncFunc func()) {
@@ -62,3 +69,7 @@ func RunSyncCloudproviderRegionTask(key string, syncFunc func()) {
 func RunSyncCloudAccountTask(probeFunc func()) {
 	syncAccountWorker.Run(probeFunc, nil, nil)
 }
+
+func RunSyncSecgroupTask(syncFunc func()) {
+	syncSecgroupWorker.Run(syncFunc, nil, nil)
+}

pkg/compute/models/vpcs_mapped_address.go

@@ -39,7 +39,7 @@ func (man *SGuestnetworkManager) lockAllocMappedAddr(ctx context.Context) {
 }
 
 func (man *SGuestnetworkManager) unlockAllocMappedAddr(ctx context.Context) {
-	defer lockman.ReleaseRawObject(ctx, LOCK_CLASS_guestnetworks_mapped_addr, LOCK_OBJ_guestnetworks_mapped_addr)
+	lockman.ReleaseRawObject(ctx, LOCK_CLASS_guestnetworks_mapped_addr, LOCK_OBJ_guestnetworks_mapped_addr)
 }
 
 func (man *SGuestnetworkManager) allocMappedIpAddr(ctx context.Context) (string, error) {

pkg/compute/models/wireresource.go

@@ -192,11 +192,11 @@ func (manager *SWireResourceBaseManager) ListItemFilter(
 		}
 		err = q.First(region)
 		if err != nil {
-			return nil, errors.Wrap(err, "q.First")
+			return nil, errors.Wrap(err, "regionQ.First")
 		}
 		if utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
 			vpcQ := VpcManager.Query().SubQuery()
-			q = q.Join(vpcQ, sqlchemy.Equals(vpcQ.Field("id"), q.Field("vpc_id"))).
+			wireQ = wireQ.Join(vpcQ, sqlchemy.Equals(vpcQ.Field("id"), wireQ.Field("vpc_id"))).
 				Filter(sqlchemy.Equals(vpcQ.Field("cloudregion_id"), region.Id))
 		} else {
 			zoneQuery := api.ZonalFilterListInput{

pkg/compute/models/wires.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -362,8 +363,9 @@ func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.Toke
 
 		if self.IsEmulated {
 			self.DomainId = vpc.DomainId
-			self.IsPublic = vpc.IsPublic
-			self.PublicScope = vpc.PublicScope
+			// self.IsPublic = vpc.IsPublic
+			// self.PublicScope = vpc.PublicScope
+			// self.PublicSrc = vpc.PublicSrc
 		}
 
 		return nil
@@ -375,6 +377,8 @@ func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.Toke
 	if provider != nil && !self.IsEmulated {
 		SyncCloudDomain(userCred, self, provider.GetOwnerId())
 		self.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
+	} else if self.IsEmulated {
+		self.SaveSharedInfo(apis.TOwnerSource(vpc.PublicSrc), ctx, userCred, vpc.GetSharedInfo())
 	}
 
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
@@ -418,6 +422,7 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl
 	wire.DomainId = vpc.DomainId
 	wire.IsPublic = vpc.IsPublic
 	wire.PublicScope = vpc.PublicScope
+	wire.PublicSrc = vpc.PublicSrc
 
 	err = manager.TableSpec().Insert(ctx, &wire)
 	if err != nil {
@@ -454,6 +459,7 @@ func (manager *SWireManager) totalCountQ(
 ) *sqlchemy.SQuery {
 	guestsQ := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId)
 	guests := guestsQ.SubQuery()
+
 	hostsQ := HostManager.Query()
 	if len(hostTypes) > 0 {
 		hostsQ = hostsQ.In("host_type", hostTypes)
@@ -465,7 +471,9 @@ func (manager *SWireManager) totalCountQ(
 		hostsQ = RangeObjectsFilter(hostsQ, rangeObjs, nil, hostsQ.Field("zone_id"), hostsQ.Field("manager_id"), hostsQ.Field("id"), nil)
 	}
 	hosts := hostsQ.SubQuery()
+
 	groups := filterByScopeOwnerId(GroupManager.Query(), scope, ownerId).SubQuery()
+
 	lbsQ := filterByScopeOwnerId(LoadbalancerManager.Query(), scope, ownerId)
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		lbsQ = CloudProviderFilter(lbsQ, lbsQ.Field("manager_id"), providers, brands, cloudEnv)
@@ -610,21 +618,20 @@ func (manager *SWireManager) totalCountQ(
 	)
 	q = q.LeftJoin(netSQ, sqlchemy.Equals(wires.Field("id"), netSQ.Field("wire_id")))
 
-	if len(rangeObjs) > 0 || len(hostTypes) > 0 {
+	if len(hostTypes) > 0 {
 		hostwires := HostwireManager.Query().SubQuery()
-		sq := hostwires.Query(hostwires.Field("wire_id"))
-		sq = sq.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id")))
-		sq = sq.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-		sq = AttachUsageQuery(sq, hosts, hostTypes, nil, nil, nil, "", rangeObjs)
-		q = q.Filter(sqlchemy.In(wires.Field("id"), sq.Distinct()))
+		hosts := HostManager.Query().SubQuery()
+		q = q.Join(hostwires, sqlchemy.Equals(q.Field("id"), hostwires.Field("wire_id")))
+		q = q.Join(hosts, sqlchemy.Equals(hostwires.Field("host_id"), hosts.Field("id")))
+		q = q.Filter(sqlchemy.In(hosts.Field("host_type"), hostTypes))
 	}
 
-	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+	if len(rangeObjs) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		vpcs := VpcManager.Query().SubQuery()
 
-		subq := vpcs.Query(vpcs.Field("id"))
-		subq = CloudProviderFilter(subq, vpcs.Field("manager_id"), providers, brands, cloudEnv)
-		q = q.Filter(sqlchemy.In(wires.Field("vpc_id"), subq.SubQuery()))
+		q = q.Join(vpcs, sqlchemy.Equals(q.Field("vpc_id"), vpcs.Field("id")))
+		q = CloudProviderFilter(q, vpcs.Field("manager_id"), providers, brands, cloudEnv)
+		q = RangeObjectsFilter(q, rangeObjs, vpcs.Field("cloudregion_id"), q.Field("zone_id"), vpcs.Field("manager_id"), nil, nil)
 	}
 
 	return q

pkg/compute/options/options.go

@@ -140,10 +140,14 @@ type ComputeOptions struct {
 
 	ReconcileGuestBackupIntervalSeconds int `help:"interval reconcile guest bakcups" default:"30"`
 
+	EnableAutoRenameProject bool `help:"when it set true, auto create project will rename when cloud project name changed" default:"false"`
+
 	SCapabilityOptions
 	SASControllerOptions
 	common_options.CommonOptions
 	common_options.DBOptions
+
+	EnableAutoMergeSecurityGroup bool `help:"Enable auto merge secgroup when sync security group from cloud, default False" default:"false"`
 }
 
 type SCapabilityOptions struct {

pkg/compute/regiondrivers/aws.go

@@ -60,7 +60,7 @@ func (self *SAwsRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.Secu
 }
 
 func (self *SAwsRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
-	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
 }
 
 func (self *SAwsRegionDriver) GetSecurityGroupRuleMaxPriority() int {

pkg/compute/regiondrivers/base.go

@@ -17,7 +17,6 @@ package regiondrivers
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -302,10 +301,6 @@ func (self *SBaseRegionDriver) InitDBInstanceUser(ctx context.Context, dbinstanc
 	return nil
 }
 
-func (self *SBaseRegionDriver) RequestRenewDBInstance(instance *models.SDBInstance, bc billing.SBillingCycle) (time.Time, error) {
-	return time.Time{}, fmt.Errorf("Not Implement RequestRenewDBInstance")
-}
-
 func (self *SBaseRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement RequestChangeDBInstanceConfig")
 }

pkg/compute/regiondrivers/huawei.go

@@ -1914,7 +1914,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancer(ctx context.Context,
 				}
 
 				eip, err := db.FetchByExternalIdAndManagerId(models.ElasticipManager, ieip.GetGlobalId(), func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
-					return q.Equals("manager_id", lb.MarkUnDelete)
+					return q.Equals("manager_id", lb.ManagerId)
 				})
 				if err != nil {
 					return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.FetchByExternalId")

pkg/compute/regiondrivers/kvm.go

@@ -894,13 +894,13 @@ func (self *SKVMRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mcc
 }
 
 func (self *SKVMRegionDriver) ValidateCreateEipData(ctx context.Context, userCred mcclient.TokenCredential, input *api.SElasticipCreateInput) error {
-	if len(input.Network) == 0 {
-		return httperrors.NewMissingParameterError("network")
+	if len(input.NetworkId) == 0 {
+		return httperrors.NewMissingParameterError("network_id")
 	}
-	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.Network)
+	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.NetworkId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			return httperrors.NewResourceNotFoundError("failed to found network %s", input.Network)
+			return httperrors.NewResourceNotFoundError2("network", input.NetworkId)
 		}
 		return httperrors.NewGeneralError(err)
 	}
@@ -918,7 +918,7 @@ func (self *SKVMRegionDriver) ValidateCreateEipData(ctx context.Context, userCre
 	if err != nil {
 		return err
 	}
-	if region.GetDriver().GetProvider() != self.GetProvider() {
+	if region.Id != input.CloudregionId {
 		return httperrors.NewUnsupportOperationError("network %s(%s) does not belong to %s", network.Name, network.Id, self.GetProvider())
 	}
 	return nil

pkg/compute/regiondrivers/managedvirtual.go

@@ -1476,95 +1476,109 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 		return "", errors.Wrap(err, "SSecurityGroupCache.Register")
 	}
 
-	iRegion, err := vpc.GetIRegion()
-	if err != nil {
-		return "", errors.Wrap(err, "vpc.GetIRegion")
-	}
+	waitChan := make(chan error)
 
-	var iSecgroup cloudprovider.ICloudSecurityGroup = nil
-	if len(cache.ExternalId) > 0 {
-		iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
-		if err != nil {
-			if errors.Cause(err) != cloudprovider.ErrNotFound {
-				return "", errors.Wrap(err, "iRegion.GetSecurityGroupById")
-			}
-			cache.ExternalId = ""
-		}
-	}
-
-	if len(cache.ExternalId) == 0 {
-		if strings.ToLower(secgroup.Name) == "default" { //避免有些云不支持default关键字
-			secgroup.Name = "DefaultGroup"
-		}
-		// 避免有的云不支持重名安全组
-		randomString := func(prefix string, length int) string {
-			return fmt.Sprintf("%s-%s", prefix, rand.String(length))
-		}
-		opts := &cloudprovider.SecurityGroupFilterOptions{
-			Name:      randomString(secgroup.Name, 1),
-			VpcId:     vpcId,
-			ProjectId: remoteProjectId,
-		}
-		for i := 2; i < 30; i++ {
-			_, err := iRegion.GetISecurityGroupByName(opts)
+	models.RunSyncSecgroupTask(func() {
+		err := func() error {
+			iRegion, err := vpc.GetIRegion()
 			if err != nil {
-				if errors.Cause(err) == cloudprovider.ErrNotFound {
-					break
-				}
-				if errors.Cause(err) != cloudprovider.ErrDuplicateId {
-					return "", err
+				return errors.Wrap(err, "vpc.GetIRegion")
+			}
+
+			var iSecgroup cloudprovider.ICloudSecurityGroup = nil
+			if len(cache.ExternalId) > 0 {
+				iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
+				if err != nil {
+					if errors.Cause(err) != cloudprovider.ErrNotFound {
+						return errors.Wrap(err, "iRegion.GetSecurityGroupById")
+					}
+					cache.ExternalId = ""
 				}
 			}
-			opts.Name = randomString(secgroup.Name, i)
-		}
-		conf := &cloudprovider.SecurityGroupCreateInput{
-			Name:      opts.Name,
-			Desc:      secgroup.Description,
-			VpcId:     vpcId,
-			ProjectId: remoteProjectId,
-			Rules:     secgroup.GetSecRules(""),
-		}
-		iSecgroup, err = iRegion.CreateISecurityGroup(conf)
-		if err != nil {
-			return "", errors.Wrap(err, "iRegion.CreateISecurityGroup")
-		}
-	}
 
-	_, err = db.Update(cache, func() error {
-		cache.ExternalId = iSecgroup.GetGlobalId()
-		cache.Name = iSecgroup.GetName()
-		cache.Status = api.SECGROUP_CACHE_STATUS_READY
-		return nil
+			if len(cache.ExternalId) == 0 {
+				if strings.ToLower(secgroup.Name) == "default" { //避免有些云不支持default关键字
+					secgroup.Name = "DefaultGroup"
+				}
+				// 避免有的云不支持重名安全组
+				randomString := func(prefix string, length int) string {
+					return fmt.Sprintf("%s-%s", prefix, rand.String(length))
+				}
+				opts := &cloudprovider.SecurityGroupFilterOptions{
+					Name:      randomString(secgroup.Name, 1),
+					VpcId:     vpcId,
+					ProjectId: remoteProjectId,
+				}
+				for i := 2; i < 30; i++ {
+					_, err := iRegion.GetISecurityGroupByName(opts)
+					if err != nil {
+						if errors.Cause(err) == cloudprovider.ErrNotFound {
+							break
+						}
+						if errors.Cause(err) != cloudprovider.ErrDuplicateId {
+							return errors.Wrapf(err, "GetISecurityGroupByName")
+						}
+					}
+					opts.Name = randomString(secgroup.Name, i)
+				}
+				conf := &cloudprovider.SecurityGroupCreateInput{
+					Name:      opts.Name,
+					Desc:      secgroup.Description,
+					VpcId:     vpcId,
+					ProjectId: remoteProjectId,
+					Rules:     secgroup.GetSecRules(""),
+				}
+				iSecgroup, err = iRegion.CreateISecurityGroup(conf)
+				if err != nil {
+					return errors.Wrapf(err, "iRegion.CreateISecurityGroup")
+				}
+			}
+
+			_, err = db.Update(cache, func() error {
+				cache.ExternalId = iSecgroup.GetGlobalId()
+				cache.Name = iSecgroup.GetName()
+				cache.Status = api.SECGROUP_CACHE_STATUS_READY
+				return nil
+			})
+
+			if err != nil {
+				return errors.Wrapf(err, "db.Update")
+			}
+
+			rules, err := iSecgroup.GetRules()
+			if err != nil {
+				return errors.Wrapf(err, "iSecgroup.GetRules")
+			}
+
+			maxPriority := region.GetDriver().GetSecurityGroupRuleMaxPriority()
+			minPriority := region.GetDriver().GetSecurityGroupRuleMinPriority()
+
+			defaultInRule := region.GetDriver().GetDefaultSecurityGroupInRule()
+			defaultOutRule := region.GetDriver().GetDefaultSecurityGroupOutRule()
+			order := region.GetDriver().GetSecurityGroupRuleOrder()
+			onlyAllowRules := region.GetDriver().IsOnlySupportAllowRules()
+
+			localRules := secrules.SecurityRuleSet(secgroup.GetSecRules(""))
+
+			common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
+
+			if len(inAdds) == 0 && len(inDels) == 0 && len(outAdds) == 0 && len(outDels) == 0 {
+				return nil
+			}
+			return iSecgroup.SyncRules(common, inAdds, outAdds, inDels, outDels)
+		}()
+
+		waitChan <- err
 	})
 
+	err = <-waitChan
 	if err != nil {
-		return "", errors.Wrap(err, "db.Update")
+		return "", err
 	}
 
-	rules, err := iSecgroup.GetRules()
+	cache, err = models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId, remoteProjectId)
 	if err != nil {
-		return "", errors.Wrap(err, "iSecgroup.GetRules")
-	}
-
-	maxPriority := region.GetDriver().GetSecurityGroupRuleMaxPriority()
-	minPriority := region.GetDriver().GetSecurityGroupRuleMinPriority()
-
-	defaultInRule := region.GetDriver().GetDefaultSecurityGroupInRule()
-	defaultOutRule := region.GetDriver().GetDefaultSecurityGroupOutRule()
-	order := region.GetDriver().GetSecurityGroupRuleOrder()
-	onlyAllowRules := region.GetDriver().IsOnlySupportAllowRules()
-
-	localRules := secrules.SecurityRuleSet(secgroup.GetSecRules(""))
-
-	common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
-
-	if len(inAdds) == 0 && len(inDels) == 0 && len(outAdds) == 0 && len(outDels) == 0 {
-		return cache.ExternalId, nil
-	}
-
-	err = iSecgroup.SyncRules(common, inAdds, outAdds, inDels, outDels)
-	if err != nil {
-		return "", errors.Wrap(err, "iSecgroup.SyncRules")
+		return "", errors.Wrap(err, "SSecurityGroupCache.Register")
 	}
 
 	return cache.ExternalId, nil

pkg/compute/regiondrivers/openstack.go

@@ -87,13 +87,13 @@ func (self *SOpenStackRegionDriver) ValidateCreateLoadbalancerCertificateData(ct
 }
 
 func (self *SOpenStackRegionDriver) ValidateCreateEipData(ctx context.Context, userCred mcclient.TokenCredential, input *api.SElasticipCreateInput) error {
-	if len(input.Network) == 0 {
-		return httperrors.NewMissingParameterError("network")
+	if len(input.NetworkId) == 0 {
+		return httperrors.NewMissingParameterError("network_id")
 	}
-	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.Network)
+	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.NetworkId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			return httperrors.NewResourceNotFoundError("failed to found network %s", input.Network)
+			return httperrors.NewResourceNotFoundError2("network", input.NetworkId)
 		}
 		return httperrors.NewGeneralError(err)
 	}
@@ -104,11 +104,12 @@ func (self *SOpenStackRegionDriver) ValidateCreateEipData(ctx context.Context, u
 	if vpc == nil {
 		return httperrors.NewInputParameterError("failed to found vpc for network %s(%s)", network.Name, network.Id)
 	}
+	input.ManagerId = vpc.ManagerId
 	region, err := vpc.GetRegion()
 	if err != nil {
 		return err
 	}
-	if region.GetDriver().GetProvider() != self.GetProvider() {
+	if region.Id != input.CloudregionId {
 		return httperrors.NewUnsupportOperationError("network %s(%s) does not belong to %s", network.Name, network.Id, self.GetProvider())
 	}
 	return nil

pkg/compute/regiondrivers/secgroup_aws_test.go

@@ -34,7 +34,7 @@ func TestAwsRuleSync(t *testing.T) {
 
 	data := []TestData{
 		{
-			Name: "Test out deny rules",
+			Name: "Test remove out allow rules",
 			LocalRules: secrules.SecurityRuleSet{
 				localRuleWithPriority("out:deny any", 1),
 			},
@@ -49,6 +49,30 @@ func TestAwsRuleSync(t *testing.T) {
 				remoteRuleWithName("", "out:allow any", 1),
 			},
 		},
+		{
+			Name: "Test out deny rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds:     []cloudprovider.SecurityRule{},
+			InDels:      []cloudprovider.SecurityRule{},
+			OutDels:     []cloudprovider.SecurityRule{},
+		},
+		{
+			Name:        "Test out allow rules",
+			LocalRules:  secrules.SecurityRuleSet{},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 0),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
 	}
 
 	for _, d := range data {

pkg/compute/regiondrivers/secgroup_qcloud_test.go

@@ -0,0 +1,68 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regiondrivers
+
+import (
+	"sort"
+	"testing"
+
+	"yunion.io/x/pkg/util/secrules"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+func TestQcloudRuleSync(t *testing.T) {
+	driver := SQcloudRegionDriver{}
+	maxPriority := driver.GetSecurityGroupRuleMaxPriority()
+	minPriority := driver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := driver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := driver.GetDefaultSecurityGroupOutRule()
+	order := driver.GetSecurityGroupRuleOrder()
+	isOnlyAllowRules := driver.IsOnlySupportAllowRules()
+
+	data := []TestData{
+		{
+			Name: "Test out rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:allow any", 11),
+				localRuleWithPriority("out:deny any", 10),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 100),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+	}
+
+	for _, d := range data {
+		t.Logf("check %s", d.Name)
+		common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, d.LocalRules, d.RemoteRules, defaultInRule, defaultOutRule, isOnlyAllowRules, true)
+		sort.Sort(cloudprovider.SecurityRuleSet(common))
+		sort.Sort(cloudprovider.SecurityRuleSet(inAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(outAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(inDels))
+		sort.Sort(cloudprovider.SecurityRuleSet(outDels))
+		check(t, "common", common, d.Common)
+		check(t, "inAdds", inAdds, d.InAdds)
+		check(t, "outAdds", outAdds, d.OutAdds)
+		check(t, "inDels", inDels, d.InDels)
+		check(t, "outDels", outDels, d.OutDels)
+	}
+}

pkg/compute/regiondrivers/zstack.go

@@ -78,13 +78,13 @@ func (self *SZStackRegionDriver) ValidateCreateLoadbalancerCertificateData(ctx c
 }
 
 func (self *SZStackRegionDriver) ValidateCreateEipData(ctx context.Context, userCred mcclient.TokenCredential, input *api.SElasticipCreateInput) error {
-	if len(input.Network) == 0 {
-		return httperrors.NewMissingParameterError("network")
+	if len(input.NetworkId) == 0 {
+		return httperrors.NewMissingParameterError("network_id")
 	}
-	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.Network)
+	_network, err := models.NetworkManager.FetchByIdOrName(userCred, input.NetworkId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			return httperrors.NewResourceNotFoundError("failed to found network %s", input.Network)
+			return httperrors.NewResourceNotFoundError2("network", input.NetworkId)
 		}
 		return httperrors.NewGeneralError(err)
 	}
@@ -95,11 +95,12 @@ func (self *SZStackRegionDriver) ValidateCreateEipData(ctx context.Context, user
 	if vpc == nil {
 		return httperrors.NewInputParameterError("failed to found vpc for network %s(%s)", network.Name, network.Id)
 	}
+	input.ManagerId = vpc.ManagerId
 	region, err := vpc.GetRegion()
 	if err != nil {
 		return err
 	}
-	if region.GetDriver().GetProvider() != self.GetProvider() {
+	if region.Id != input.CloudregionId {
 		return httperrors.NewUnsupportOperationError("network %s(%s) does not belong to %s", network.Name, network.Id, self.GetProvider())
 	}
 	return nil

pkg/compute/tasks/cloud_account_delete_task.go

@@ -16,8 +16,11 @@ package tasks
 
 import (
 	"context"
+	"database/sql"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -62,6 +65,18 @@ func (self *CloudAccountDeleteTask) OnInit(ctx context.Context, obj db.IStandalo
 func (self *CloudAccountDeleteTask) OnAllCloudProviderDeleteComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
 	account := obj.(*models.SCloudaccount)
 
+	// check providers deleted success
+	count, err := account.GetProviderCount()
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		self.OnAllCloudProviderDeleteCompleteFailed(ctx, obj, jsonutils.NewString(err.Error()))
+		return
+	}
+
+	if count > 0 {
+		self.OnAllCloudProviderDeleteCompleteFailed(ctx, obj, jsonutils.NewString(fmt.Sprintf("cloudprovider deleted failed count %d", count)))
+		return
+	}
+
 	account.RealDelete(ctx, self.UserCred)
 
 	self.SetStageComplete(ctx, nil)