Merge pull request #8528 from rainzm/automated-cherry-pick-of-#8525-upstream-release-3.3

Automated cherry pick of #8525: fix(esxi): no callback in uploadHandler

.circleci/config.yml

@@ -38,7 +38,7 @@ jobs:
           command: |
             targets="$(circleci tests glob "cmd/*" | grep -v cmd/host-image | circleci tests split)"
             echo $targets | tr ' ' '\n'
-            make -j3 $targets
+            make -j2 $targets
             ls -lh _output/bin/
       - save_cache:
           key: onecloud-build-cache-{{ checksum "go.mod" }}-{{ checksum "Makefile" }}

go.mod

@@ -131,12 +131,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8
+	yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
 	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6
+	yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -1126,8 +1126,8 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8 h1:KQVw5EmohJCqBDMdZ3fAcOQNNMIYgVuNCNWNwg4EM3A=
-yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6 h1:RPpnTSCZwa9VQuWAf7S2dRjnZlYTIM0NH/j5vvUpk9g=
+yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
@@ -1140,7 +1140,7 @@ yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6 h1:k7E8Zjwzyk/lo1dDO0jtLbODwYRNv+w/zsUJrKWCXvo=
-yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8 h1:AkPrqO37Yi3aICys9qDjTlSdVVGh2pWQ8ET+dSJmhoI=
+yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/storage_const.go

@@ -81,6 +81,7 @@ const (
 	STORAGE_GOOGLE_LOCAL_SSD   = "local-ssd"   //本地SSD暂存盘 (最多8个)
 	STORAGE_GOOGLE_PD_STANDARD = "pd-standard" //标准永久性磁盘
 	STORAGE_GOOGLE_PD_SSD      = "pd-ssd"      //SSD永久性磁盘
+	STORAGE_GOOGLE_PD_BALANCED = "pd-balanced" //平衡永久性磁盘
 
 	// ctyun storage type
 	STORAGE_CTYUN_SSD  = "SSD"  // 超高IO云硬盘

pkg/cloudcommon/db/caller.go

@@ -302,6 +302,7 @@ func FetchCustomizeColumns(
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
+		// NOTE: don't use obj update jsonDict as retval
 		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
 		retVal[i] = jsonDict
 	}

pkg/cloudcommon/db/domain.go

@@ -107,12 +107,7 @@ func (manager *SDomainizedResourceBaseManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	if len(query.ProjectDomains) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.ProjectDomains {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.ProjectDomains)
 		tenants := TenantCacheManager.GetDomainQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/db/opslog.go

@@ -288,31 +288,29 @@ type SOpsLogManager struct {
 type SOpsLog struct {
 	SModelBase
 
-	Id      int64  `primary:"true" auto_increment:"true" list:"user"`                                        // = Column(BigInteger, primary_key=True)
-	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`               // = Column(VARCHAR(40, charset='ascii'), nullable=False)
-	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
-	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`               //= Column(VARCHAR(128, charset='utf8'), nullable=False)
-	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`                //= Column(VARCHAR(32, charset='ascii'), nullable=False)
+	Id      int64  `primary:"true" auto_increment:"true" list:"user"`
+	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
+	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`
+	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`
 	Notes   string `charset:"utf8" list:"user" create:"required"`
 
-	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"required" index:"true"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"required"`                  // tenant    = Column(VARCHAR(128, charset='utf8'))
+	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"optional" index:"true"`
+	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"required"`
-	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"required"`
+	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
+	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	User     string `width:"128" charset:"utf8" list:"user" create:"required"`  // = Column(VARCHAR(128, charset='utf8'))
+	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"`
+	User     string `width:"128" charset:"utf8" list:"user" create:"required"`
 	DomainId string `width:"128" charset:"ascii" list:"user" create:"optional"`
 	Domain   string `width:"128" charset:"utf8" list:"user" create:"optional"`
-	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(64, charset='ascii'))
+	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"`
 
-	// BillingType    string    `width:"64" charset:"ascii" default:"postpaid" list:"user" create:"user"`      // billing_type = Column(VARCHAR(64, charset='ascii'), nullable=True)
-	OpsTime time.Time `nullable:"false" list:"user"` // = Column(DateTime, nullable=False)
+	OpsTime time.Time `nullable:"false" list:"user"`
 
 	OwnerDomainId  string `name:"owner_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
-	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	// owner_user_id   = Column(VARCHAR(ID_LENGTH, charset='ascii'))
+	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"`
 }
 
 var OpsLog *SOpsLogManager
@@ -677,6 +675,7 @@ func (manager *SOpsLogManager) ValidateCreateData(ctx context.Context,
 	query jsonutils.JSONObject,
 	data apis.OpsLogCreateInput,
 ) (apis.OpsLogCreateInput, error) {
+	data.User = ownerId.GetUserName()
 	return data, nil
 }
 

pkg/cloudcommon/db/project.go

@@ -102,12 +102,7 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 	}
 	if len(query.Projects) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.Projects {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.Projects)
 		tenants := TenantCacheManager.GetTenantQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/quotas/register.go

@@ -33,6 +33,7 @@ var (
 func init() {
 	quotaManagerTable = make(map[reflect.Type]IQuotaManager)
 
+	db.AddUsages = AddUsages
 	db.CancelUsages = CancelUsages
 	db.CancelPendingUsagesInContext = cancelPendingUsagesInContext
 	db.InitPendingUsagesInContext = initPendingUsagesInContext

pkg/cloudcommon/db/usages.go

@@ -28,6 +28,8 @@ type IUsage interface {
 var (
 	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
 
+	AddUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
+
 	CancelPendingUsagesInContext func(ctx context.Context, userCred mcclient.TokenCredential) error
 
 	InitPendingUsagesInContext func(ctx context.Context) context.Context

pkg/cloudcommon/db/virtualresource.go

@@ -393,11 +393,21 @@ func (model *SVirtualResourceBase) AllowPerformCancelDelete(ctx context.Context,
 func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if model.PendingDeleted && !model.Deleted {
 		err := model.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "model.DoCancelPendingDelete")
+		}
+		model.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
 
+func (model *SVirtualResourceBase) RecoverUsages(ctx context.Context, userCred mcclient.TokenCredential) {
+	usages := model.GetIModel().GetUsages()
+	if AddUsages != nil && len(usages) > 0 {
+		AddUsages(ctx, userCred, usages)
+	}
+}
+
 func (model *SVirtualResourceBase) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	err := model.CancelPendingDelete(ctx, userCred)
 	if err == nil {
@@ -416,7 +426,10 @@ func (model *SVirtualResourceBase) GetIVirtualModel() IVirtualModel {
 
 func (model *SVirtualResourceBase) CancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	if model.PendingDeleted && !model.Deleted {
-		return model.MarkCancelPendingDelete(ctx, userCred)
+		err := model.MarkCancelPendingDelete(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "MarkCancelPendingDelete")
+		}
 	}
 	return nil
 }

pkg/cloudprovider/securitygroup.go

@@ -23,6 +23,7 @@ import (
 )
 
 const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
+const DEFAULT_LOCAL_RULE_ID = "default_local_rule_id"
 
 type SecurityGroupFilterOptions struct {
 	VpcId     string
@@ -44,6 +45,42 @@ type SecurityRule struct {
 	ExternalId string
 }
 
+type LocalSecurityRule struct {
+	secrules.SecurityRule
+	ExternalId string
+}
+
+func (r LocalSecurityRule) String() string {
+	return r.SecurityRule.String()
+}
+
+type LocalSecurityRuleSet []LocalSecurityRule
+
+func (srs LocalSecurityRuleSet) Len() int {
+	return len(srs)
+}
+
+func (srs LocalSecurityRuleSet) Swap(i, j int) {
+	srs[i], srs[j] = srs[j], srs[i]
+}
+
+func (srs LocalSecurityRuleSet) Less(i, j int) bool {
+	if srs[i].Priority > srs[j].Priority {
+		return true
+	} else if srs[i].Priority == srs[j].Priority {
+		return srs[i].String() < srs[j].String()
+	}
+	return false
+}
+
+func (srs LocalSecurityRuleSet) AllowList() secrules.SecurityRuleSet {
+	rules := secrules.SecurityRuleSet{}
+	for _, r := range srs {
+		rules = append(rules, r.SecurityRule)
+	}
+	return rules.AllowList()
+}
+
 type TPriorityOrder int
 
 var (
@@ -112,13 +149,15 @@ func CompareRules(
 	defaultInRule, defaultOutRule SecurityRule,
 	onlyAllowRules bool, debug bool,
 ) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
-	localInRules := secrules.SecurityRuleSet{}
-	localOutRules := secrules.SecurityRuleSet{}
+	localInRules := LocalSecurityRuleSet{}
+	localOutRules := LocalSecurityRuleSet{}
 	for i := range localRules {
+		localRule := LocalSecurityRule{}
+		localRule.SecurityRule = localRules[i]
 		if localRules[i].Direction == secrules.DIR_IN {
-			localInRules = append(localInRules, localRules[i])
+			localInRules = append(localInRules, localRule)
 		} else {
-			localOutRules = append(localOutRules, localRules[i])
+			localOutRules = append(localOutRules, localRule)
 		}
 	}
 	inRules := SecurityRuleSet{}
@@ -132,17 +171,22 @@ func CompareRules(
 	}
 	var inCommon, outCommon = inRules, outRules
 
-	defaultLocalInRule := *secrules.MustParseSecurityRule("in:deny any")
-	defaultLocalOutRule := *secrules.MustParseSecurityRule("out:allow any")
+	defaultLocalInRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
+	defaultLocalInRule.SecurityRule = *secrules.MustParseSecurityRule("in:deny any")
+	defaultLocalOutRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
+	defaultLocalOutRule.SecurityRule = *secrules.MustParseSecurityRule("out:allow any")
 
 	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
 	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
 
+	defaultInEquals, defaultOutEquals := true, true
 	if defaultLocalInRule.String() != defaultInRule.String() {
 		localInRules = append(localInRules, defaultLocalInRule)
+		defaultInEquals = false
 	}
 	if defaultLocalOutRule.String() != defaultOutRule.String() {
 		localOutRules = append(localOutRules, defaultLocalOutRule)
+		defaultOutEquals = false
 	}
 
 	sort.Sort(localInRules)
@@ -150,9 +194,26 @@ func CompareRules(
 
 	localInAllowList := localInRules.AllowList()
 	localOutAllowList := localOutRules.AllowList()
+	_localInRules := LocalSecurityRuleSet{}
+	for i := range localInAllowList {
+		rule := LocalSecurityRule{}
+		rule.SecurityRule = localInAllowList[i]
+		_localInRules = append(_localInRules, rule)
+	}
+	_localOutRules := LocalSecurityRuleSet{}
+	for i := range localOutAllowList {
+		rule := LocalSecurityRule{}
+		rule.SecurityRule = localOutAllowList[i]
+		_localOutRules = append(_localOutRules, rule)
+	}
 	if onlyAllowRules {
-		localInRules = localInAllowList
-		localOutRules = localOutAllowList
+		localOutRules, localInRules = _localOutRules, _localInRules
+	}
+	if len(_localInRules) < len(localInRules) {
+		localInRules = _localInRules
+	}
+	if len(_localOutRules) < len(localOutRules) {
+		localOutRules = _localOutRules
 	}
 
 	SortSecurityRule(inRules, order, onlyAllowRules)
@@ -199,7 +260,7 @@ func CompareRules(
 		return init
 	}
 
-	var compare = func(localRules secrules.SecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
+	var compare = func(localRules LocalSecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
 		i, j, inc, prePriority := 0, 0, 1, 0
 		for i < len(localRules) || j < len(remoteRules) {
 			if i < len(localRules) && j < len(remoteRules) {
@@ -214,7 +275,11 @@ func CompareRules(
 					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
 						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
 					}
-					common = append(common, remoteRules[j])
+					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+						common = append(common, remoteRules[j])
+					}
 					i++
 					j++
 				} else if cmp < 0 {
@@ -225,7 +290,11 @@ func CompareRules(
 				} else {
 					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
 					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-					add = append(add, SecurityRule{SecurityRule: localRules[i]})
+					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+						add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
+					}
 					i++
 					inc++
 				}
@@ -241,7 +310,11 @@ func CompareRules(
 				}
 				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
 				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-				add = append(add, SecurityRule{SecurityRule: localRules[i]})
+				if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
+					(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
+					(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
+					add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
+				}
 				i++
 				inc++
 			}
@@ -250,17 +323,17 @@ func CompareRules(
 	}
 
 	type rulePair struct {
-		localRules  []secrules.SecurityRule
+		localRules  LocalSecurityRuleSet
 		remoteRules []SecurityRule
 		protocol    string
 	}
 
-	var splitRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) []rulePair {
+	var splitRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) []rulePair {
 		rules := map[string]rulePair{}
 		for _, r := range localRules {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
 			}
 			pair.localRules = append(pair.localRules, r)
 			rules[r.Protocol] = pair
@@ -269,7 +342,7 @@ func CompareRules(
 		for _, r := range remoteRules {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
 			}
 			pair.remoteRules = append(pair.remoteRules, r)
 			rules[r.Protocol] = pair
@@ -282,7 +355,7 @@ func CompareRules(
 		return ret
 	}
 
-	var compareRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
+	var compareRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
 		pairs := splitRules(localRules, remoteRules)
 		for _, r := range pairs {
 			_common, _add, _dels := compare(r.localRules, r.remoteRules)

pkg/compute/guestdrivers/google.go

@@ -77,6 +77,7 @@ func (self *SGoogleGuestDriver) GetStorageTypes() []string {
 	return []string{
 		api.STORAGE_GOOGLE_PD_SSD,
 		api.STORAGE_GOOGLE_PD_STANDARD,
+		api.STORAGE_GOOGLE_PD_BALANCED,
 		api.STORAGE_GOOGLE_LOCAL_SSD,
 	}
 }
@@ -113,7 +114,7 @@ func (self *SGoogleGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	if !utils.IsInStringArray(guest.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status)
 	}
-	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD}) {
+	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED}) {
 		return fmt.Errorf("Cannot resize %s disk", storage.StorageType)
 	}
 	return nil
@@ -132,13 +133,15 @@ func (self *SGoogleGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		minGB := -1
 		maxGB := -1
 		switch disk.Backend {
-		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 			minGB = 10
 			maxGB = 65536
 		case api.STORAGE_GOOGLE_LOCAL_SSD:
 			minGB = 375
 			maxGB = 375
 			localDisk++
+		default:
+			return nil, httperrors.NewInputParameterError("Unknown google storage type %s", disk.Backend)
 		}
 		if i == 0 && disk.Backend == api.STORAGE_GOOGLE_LOCAL_SSD {
 			return nil, httperrors.NewInputParameterError("System disk does not support %s disk", disk.Backend)

pkg/compute/hostdrivers/google.go

@@ -48,7 +48,7 @@ func (self *SGoogleHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb
 	minGB := 10
 	maxGB := -1
 	switch storage.StorageType {
-	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 		maxGB = 65536
 	default:
 		return fmt.Errorf("Not support resize %s disk", storage.StorageType)

pkg/compute/models/cloudregions.go

@@ -117,6 +117,22 @@ func (self *SCloudregion) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudregion) GetElasticIps(managerId, eipMode string) ([]SElasticip, error) {
+	q := ElasticipManager.Query().Equals("cloudregion_id", self.Id)
+	if len(managerId) > 0 {
+		q = q.Equals("manager_id", managerId)
+	}
+	if len(eipMode) > 0 {
+		q = q.Equals("mode", eipMode)
+	}
+	eips := []SElasticip{}
+	err := db.FetchModelObjects(ElasticipManager, q, &eips)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return eips, nil
+}
+
 func (self *SCloudregion) GetZoneQuery() *sqlchemy.SQuery {
 	zones := ZoneManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {

pkg/compute/models/dbinstance_skus.go

@@ -164,6 +164,9 @@ func (manager *SDBInstanceSkuManager) ListItemFilter(
 	if len(query.EngineVersion) > 0 {
 		q = q.In("engine_version", query.EngineVersion)
 	}
+	if len(query.Providers) > 0 {
+		q = q.In("provider", query.Providers)
+	}
 
 	for k, zoneIds := range map[string][]string{"zone1": query.Zone1, "zone2": query.Zone2, "zone3": query.Zone3} {
 		ids := []string{}

pkg/compute/models/dbinstances.go

@@ -1536,7 +1536,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extInstance.GetBillingType()
-			self.ExpiredAt = extInstance.GetExpiredAt()
+			if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extInstance.IsAutoRenew()
 		}
 
@@ -1626,7 +1628,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 
@@ -1642,17 +1646,31 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	return &instance, nil
 }
 
+type SRdsCountStat struct {
+	TotalRdsCount  int
+	TotalCpuCount  int
+	TotalMemSizeMb int
+}
+
 func (man *SDBInstanceManager) TotalCount(
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel,
 	providers []string, brands []string, cloudEnv string,
-) (int, error) {
-	q := man.Query()
+) (SRdsCountStat, error) {
+	sq := man.Query().SubQuery()
+	q := sq.Query(sqlchemy.COUNT("total_rds_count"),
+		sqlchemy.SUM("total_cpu_count", sq.Field("vcpu_count")),
+		sqlchemy.SUM("total_mem_size_mb", sq.Field("vmem_size_mb")))
+
 	q = scopeOwnerIdFilter(q, scope, ownerId)
 	q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv)
 	q = RangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id"), nil, nil)
-	return q.CountWithError()
+
+	stat := SRdsCountStat{}
+	row := q.Row()
+	err := q.Row2Struct(row, &stat)
+	return stat, err
 }
 
 func (dbinstance *SDBInstance) GetQuotaKeys() quotas.IQuotaKeys {

pkg/compute/models/disks.go

@@ -1537,7 +1537,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		disk.BillingType = extDisk.GetBillingType()
-		disk.ExpiredAt = extDisk.GetExpiredAt()
+		if expired := extDisk.GetExpiredAt(); !expired.IsZero() {
+			disk.ExpiredAt = expired
+		}
 		disk.AutoRenew = extDisk.IsAutoRenew()
 	}
 
@@ -2142,9 +2144,12 @@ func (self *SDisk) AllowPerformCancelDelete(ctx context.Context, userCred mcclie
 }
 
 func (self *SDisk) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }

pkg/compute/models/elasticcache_instances.go

@@ -649,7 +649,9 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/elasticips.go

@@ -248,19 +248,6 @@ func (manager *SElasticipManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi
 	return q, httperrors.ErrNotFound
 }
 
-func (manager *SElasticipManager) getEipsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SElasticip, error) {
-	eips := make([]SElasticip, 0)
-	q := manager.Query().Equals("cloudregion_id", region.Id)
-	if provider != nil {
-		q = q.Equals("manager_id", provider.Id)
-	}
-	err := db.FetchModelObjects(manager, q, &eips)
-	if err != nil {
-		return nil, err
-	}
-	return eips, nil
-}
-
 func (self *SElasticip) GetRegion() *SCloudregion {
 	return CloudregionManager.FetchRegionById(self.CloudregionId)
 }
@@ -329,7 +316,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 	// remoteEips := make([]cloudprovider.ICloudEIP, 0)
 	syncResult := compare.SyncResult{}
 
-	dbEips, err := manager.getEipsByRegion(region, provider)
+	dbEips, err := region.GetElasticIps(provider.Id, api.EIP_MODE_STANDALONE_EIP)
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -478,7 +465,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.
 		factory, _ := provider.GetProviderFactory()
 		if factory != nil && factory.IsSupportPrepaidResources() {
 			self.BillingType = ext.GetBillingType()
-			self.ExpiredAt = ext.GetExpiredAt()
+			if expired := ext.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = ext.IsAutoRenew()
 		}
 

pkg/compute/models/guest_actions.go

@@ -2587,9 +2587,12 @@ func (model *SGuest) AllowPerformCancelDelete(ctx context.Context, userCred mccl
 }
 
 func (self *SGuest) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "DoCancelPendingDelete")
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }

pkg/compute/models/guests.go

@@ -2539,7 +2539,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		guest.BillingType = extVM.GetBillingType()
-		guest.ExpiredAt = extVM.GetExpiredAt()
+		if expired := extVM.GetExpiredAt(); !expired.IsZero() {
+			guest.ExpiredAt = expired
+		}
 		if guest.GetDriver().IsSupportSetAutoRenew() {
 			guest.AutoRenew = extVM.IsAutoRenew()
 		}

pkg/compute/models/managedresource.go

@@ -518,7 +518,7 @@ func _managedResourceFilterByAccount(managerIdFieldName string, q *sqlchemy.SQue
 	if len(cloudaccountArr) > 0 {
 		cpq := CloudaccountManager.Query().SubQuery()
 		subcpq := cpq.Query(cpq.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(cpq.Field("id"), cloudaccountArr),
+			sqlchemy.In(cpq.Field("id"), stringutils2.RemoveUtf8Strings(cloudaccountArr)),
 			sqlchemy.In(cpq.Field("name"), cloudaccountArr),
 		)).SubQuery()
 		subq := CloudproviderManager.Query("id").In("cloudaccount_id", subcpq).SubQuery()

pkg/compute/models/natgateways.go

@@ -363,7 +363,9 @@ func (self *SNatGateway) SyncWithCloudNatGateway(ctx context.Context, userCred m
 		factory, _ := provider.GetProviderFactory()
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extNat.GetBillingType()
-			self.ExpiredAt = extNat.GetExpiredAt()
+			if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extNat.IsAutoRenew()
 		}
 
@@ -407,7 +409,9 @@ func (manager *SNatGatewayManager) newFromCloudNatGateway(ctx context.Context, u
 	factory, _ := provider.GetProviderFactory()
 	if factory.IsSupportPrepaidResources() {
 		nat.BillingType = extNat.GetBillingType()
-		nat.ExpiredAt = extNat.GetExpiredAt()
+		if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+			nat.ExpiredAt = expired
+		}
 		nat.AutoRenew = extNat.IsAutoRenew()
 	}
 

pkg/compute/models/regionquota.go

@@ -196,7 +196,8 @@ func (self *SRegionQuota) FetchUsage(ctx context.Context) error {
 	self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000)
 	self.ObjectCnt = bucketUsage.Objects
 
-	self.Rds, _ = DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	rdsUsage, _ := DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	self.Rds = rdsUsage.TotalRdsCount
 	self.Cache, _ = ElasticcacheManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
 
 	self.Loadbalancer, _ = LoadbalancerManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)

pkg/compute/models/secgroups.go

@@ -950,6 +950,7 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, err
 	}
 
+	secgroup.Status = api.SECGROUP_STATUS_READY
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId

pkg/compute/models/snapshotpolicy.go

@@ -310,11 +310,11 @@ func (sp *SSnapshotPolicy) DetachAfterDelete(ctx context.Context, userCred mccli
 
 func (sp *SSnapshotPolicy) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	// check if sp bind to some disks
-	sds, err := SnapshotPolicyDiskManager.FetchAllBySnapshotpolicyID(ctx, userCred, sp.GetId())
+	count, err := SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	if err != nil {
-		return errors.Wrap(err, "fetch bind info failed")
+		return errors.Wrap(err, "unable to FetchDiskCountBySPID")
 	}
-	if len(sds) != 0 {
+	if count != 0 {
 		return httperrors.NewBadRequestError("Couldn't delete snapshot policy binding to disks")
 	}
 	sp.SetStatus(userCred, api.SNAPSHOT_POLICY_DELETING, "")

pkg/compute/models/syncworkers.go

@@ -26,9 +26,10 @@ import (
 )
 
 var (
-	syncAccountWorker *appsrv.SWorkerManager
-	syncWorkers       []*appsrv.SWorkerManager
-	syncWorkerRing    *hashring.HashRing
+	syncSecgroupWorker *appsrv.SWorkerManager
+	syncAccountWorker  *appsrv.SWorkerManager
+	syncWorkers        []*appsrv.SWorkerManager
+	syncWorkerRing     *hashring.HashRing
 )
 
 func InitSyncWorkers(count int) {
@@ -50,6 +51,12 @@ func InitSyncWorkers(count int) {
 		2048,
 		true,
 	)
+	syncSecgroupWorker = appsrv.NewWorkerManager(
+		"syncSecgroupProbeWorkerManager",
+		1,
+		2048,
+		true,
+	)
 }
 
 func RunSyncCloudproviderRegionTask(key string, syncFunc func()) {
@@ -62,3 +69,7 @@ func RunSyncCloudproviderRegionTask(key string, syncFunc func()) {
 func RunSyncCloudAccountTask(probeFunc func()) {
 	syncAccountWorker.Run(probeFunc, nil, nil)
 }
+
+func RunSyncSecgroupTask(syncFunc func()) {
+	syncSecgroupWorker.Run(syncFunc, nil, nil)
+}

pkg/compute/regiondrivers/managedvirtual.go

@@ -1476,95 +1476,109 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 		return "", errors.Wrap(err, "SSecurityGroupCache.Register")
 	}
 
-	iRegion, err := vpc.GetIRegion()
-	if err != nil {
-		return "", errors.Wrap(err, "vpc.GetIRegion")
-	}
+	waitChan := make(chan error)
 
-	var iSecgroup cloudprovider.ICloudSecurityGroup = nil
-	if len(cache.ExternalId) > 0 {
-		iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
-		if err != nil {
-			if errors.Cause(err) != cloudprovider.ErrNotFound {
-				return "", errors.Wrap(err, "iRegion.GetSecurityGroupById")
-			}
-			cache.ExternalId = ""
-		}
-	}
-
-	if len(cache.ExternalId) == 0 {
-		if strings.ToLower(secgroup.Name) == "default" { //避免有些云不支持default关键字
-			secgroup.Name = "DefaultGroup"
-		}
-		// 避免有的云不支持重名安全组
-		randomString := func(prefix string, length int) string {
-			return fmt.Sprintf("%s-%s", prefix, rand.String(length))
-		}
-		opts := &cloudprovider.SecurityGroupFilterOptions{
-			Name:      randomString(secgroup.Name, 1),
-			VpcId:     vpcId,
-			ProjectId: remoteProjectId,
-		}
-		for i := 2; i < 30; i++ {
-			_, err := iRegion.GetISecurityGroupByName(opts)
+	models.RunSyncSecgroupTask(func() {
+		err := func() error {
+			iRegion, err := vpc.GetIRegion()
 			if err != nil {
-				if errors.Cause(err) == cloudprovider.ErrNotFound {
-					break
-				}
-				if errors.Cause(err) != cloudprovider.ErrDuplicateId {
-					return "", err
+				return errors.Wrap(err, "vpc.GetIRegion")
+			}
+
+			var iSecgroup cloudprovider.ICloudSecurityGroup = nil
+			if len(cache.ExternalId) > 0 {
+				iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
+				if err != nil {
+					if errors.Cause(err) != cloudprovider.ErrNotFound {
+						return errors.Wrap(err, "iRegion.GetSecurityGroupById")
+					}
+					cache.ExternalId = ""
 				}
 			}
-			opts.Name = randomString(secgroup.Name, i)
-		}
-		conf := &cloudprovider.SecurityGroupCreateInput{
-			Name:      opts.Name,
-			Desc:      secgroup.Description,
-			VpcId:     vpcId,
-			ProjectId: remoteProjectId,
-			Rules:     secgroup.GetSecRules(""),
-		}
-		iSecgroup, err = iRegion.CreateISecurityGroup(conf)
-		if err != nil {
-			return "", errors.Wrap(err, "iRegion.CreateISecurityGroup")
-		}
-	}
 
-	_, err = db.Update(cache, func() error {
-		cache.ExternalId = iSecgroup.GetGlobalId()
-		cache.Name = iSecgroup.GetName()
-		cache.Status = api.SECGROUP_CACHE_STATUS_READY
-		return nil
+			if len(cache.ExternalId) == 0 {
+				if strings.ToLower(secgroup.Name) == "default" { //避免有些云不支持default关键字
+					secgroup.Name = "DefaultGroup"
+				}
+				// 避免有的云不支持重名安全组
+				randomString := func(prefix string, length int) string {
+					return fmt.Sprintf("%s-%s", prefix, rand.String(length))
+				}
+				opts := &cloudprovider.SecurityGroupFilterOptions{
+					Name:      randomString(secgroup.Name, 1),
+					VpcId:     vpcId,
+					ProjectId: remoteProjectId,
+				}
+				for i := 2; i < 30; i++ {
+					_, err := iRegion.GetISecurityGroupByName(opts)
+					if err != nil {
+						if errors.Cause(err) == cloudprovider.ErrNotFound {
+							break
+						}
+						if errors.Cause(err) != cloudprovider.ErrDuplicateId {
+							return errors.Wrapf(err, "GetISecurityGroupByName")
+						}
+					}
+					opts.Name = randomString(secgroup.Name, i)
+				}
+				conf := &cloudprovider.SecurityGroupCreateInput{
+					Name:      opts.Name,
+					Desc:      secgroup.Description,
+					VpcId:     vpcId,
+					ProjectId: remoteProjectId,
+					Rules:     secgroup.GetSecRules(""),
+				}
+				iSecgroup, err = iRegion.CreateISecurityGroup(conf)
+				if err != nil {
+					return errors.Wrapf(err, "iRegion.CreateISecurityGroup")
+				}
+			}
+
+			_, err = db.Update(cache, func() error {
+				cache.ExternalId = iSecgroup.GetGlobalId()
+				cache.Name = iSecgroup.GetName()
+				cache.Status = api.SECGROUP_CACHE_STATUS_READY
+				return nil
+			})
+
+			if err != nil {
+				return errors.Wrapf(err, "db.Update")
+			}
+
+			rules, err := iSecgroup.GetRules()
+			if err != nil {
+				return errors.Wrapf(err, "iSecgroup.GetRules")
+			}
+
+			maxPriority := region.GetDriver().GetSecurityGroupRuleMaxPriority()
+			minPriority := region.GetDriver().GetSecurityGroupRuleMinPriority()
+
+			defaultInRule := region.GetDriver().GetDefaultSecurityGroupInRule()
+			defaultOutRule := region.GetDriver().GetDefaultSecurityGroupOutRule()
+			order := region.GetDriver().GetSecurityGroupRuleOrder()
+			onlyAllowRules := region.GetDriver().IsOnlySupportAllowRules()
+
+			localRules := secrules.SecurityRuleSet(secgroup.GetSecRules(""))
+
+			common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
+
+			if len(inAdds) == 0 && len(inDels) == 0 && len(outAdds) == 0 && len(outDels) == 0 {
+				return nil
+			}
+			return iSecgroup.SyncRules(common, inAdds, outAdds, inDels, outDels)
+		}()
+
+		waitChan <- err
 	})
 
+	err = <-waitChan
 	if err != nil {
-		return "", errors.Wrap(err, "db.Update")
+		return "", err
 	}
 
-	rules, err := iSecgroup.GetRules()
+	cache, err = models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId, remoteProjectId)
 	if err != nil {
-		return "", errors.Wrap(err, "iSecgroup.GetRules")
-	}
-
-	maxPriority := region.GetDriver().GetSecurityGroupRuleMaxPriority()
-	minPriority := region.GetDriver().GetSecurityGroupRuleMinPriority()
-
-	defaultInRule := region.GetDriver().GetDefaultSecurityGroupInRule()
-	defaultOutRule := region.GetDriver().GetDefaultSecurityGroupOutRule()
-	order := region.GetDriver().GetSecurityGroupRuleOrder()
-	onlyAllowRules := region.GetDriver().IsOnlySupportAllowRules()
-
-	localRules := secrules.SecurityRuleSet(secgroup.GetSecRules(""))
-
-	common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
-
-	if len(inAdds) == 0 && len(inDels) == 0 && len(outAdds) == 0 && len(outDels) == 0 {
-		return cache.ExternalId, nil
-	}
-
-	err = iSecgroup.SyncRules(common, inAdds, outAdds, inDels, outDels)
-	if err != nil {
-		return "", errors.Wrap(err, "iSecgroup.SyncRules")
+		return "", errors.Wrap(err, "SSecurityGroupCache.Register")
 	}
 
 	return cache.ExternalId, nil

pkg/compute/regiondrivers/secgroup_qcloud_test.go

@@ -0,0 +1,68 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regiondrivers
+
+import (
+	"sort"
+	"testing"
+
+	"yunion.io/x/pkg/util/secrules"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+func TestQcloudRuleSync(t *testing.T) {
+	driver := SQcloudRegionDriver{}
+	maxPriority := driver.GetSecurityGroupRuleMaxPriority()
+	minPriority := driver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := driver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := driver.GetDefaultSecurityGroupOutRule()
+	order := driver.GetSecurityGroupRuleOrder()
+	isOnlyAllowRules := driver.IsOnlySupportAllowRules()
+
+	data := []TestData{
+		{
+			Name: "Test out rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:allow any", 11),
+				localRuleWithPriority("out:deny any", 10),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 100),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+	}
+
+	for _, d := range data {
+		t.Logf("check %s", d.Name)
+		common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, d.LocalRules, d.RemoteRules, defaultInRule, defaultOutRule, isOnlyAllowRules, true)
+		sort.Sort(cloudprovider.SecurityRuleSet(common))
+		sort.Sort(cloudprovider.SecurityRuleSet(inAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(outAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(inDels))
+		sort.Sort(cloudprovider.SecurityRuleSet(outDels))
+		check(t, "common", common, d.Common)
+		check(t, "inAdds", inAdds, d.InAdds)
+		check(t, "outAdds", outAdds, d.OutAdds)
+		check(t, "inDels", inDels, d.InDels)
+		check(t, "outDels", outDels, d.OutDels)
+	}
+}

pkg/compute/tasks/eip_dissociate_task.go

@@ -23,7 +23,6 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
@@ -85,8 +84,8 @@ func (self *EipDissociateTask) OnInit(ctx context.Context, obj db.IStandaloneMod
 			self.TaskFail(ctx, eip, "unsupported associate type", nil)
 			return
 		}
-		lockman.LockObject(ctx, model)
-		defer lockman.ReleaseObject(ctx, model)
+		// lockman.LockObject(ctx, model)
+		// defer lockman.ReleaseObject(ctx, model)
 
 		if eip.IsManaged() {
 			extEip, err := eip.GetIEip()

pkg/compute/usages/handler.go

@@ -854,7 +854,9 @@ func LoadbalancerUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityPro
 func DBInstanceUsage(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, providers []string, brands []string, cloudEnv string) Usage {
 	cnt, _ := models.DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, cloudEnv)
 	count := make(map[string]interface{})
-	count[getKey(scope, "rds")] = cnt
+	count[getKey(scope, "rds")] = cnt.TotalRdsCount
+	count[getKey(scope, "rds.cpu")] = cnt.TotalCpuCount
+	count[getKey(scope, "rds.memory")] = cnt.TotalMemSizeMb
 	return count
 }
 

pkg/esxi/handler/handlers.go

@@ -75,7 +75,7 @@ func uploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		httperrors.MissingParameterError(w, "miss disk")
 		return
 	}
-	hostutils.DelayTask(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
+	hostutils.DelayTaskWithoutReqctx(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
 	hostutils.ResponseOk(ctx, w)
 }
 

pkg/image/models/image_guest.go

@@ -246,9 +246,12 @@ func (gi *SGuestImage) AllowPerformCancelDelete(ctx context.Context, userCred mc
 func (gi *SGuestImage) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 
-	if gi.PendingDeleted {
+	if gi.PendingDeleted && !gi.Deleted {
 		err := gi.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "DoCancelPendingDelete")
+		}
+		gi.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
@@ -635,3 +638,21 @@ func (manager *SGuestImageManager) Usage(scope rbacutils.TRbacScope, ownerId mcc
 
 	return usages
 }
+
+func (gi *SGuestImage) GetUsages() []db.IUsage {
+	if gi.PendingDeleted || gi.Deleted {
+		return nil
+	}
+	images, err := GuestImageJointManager.GetImagesByGuestImageId(gi.Id)
+	if err != nil {
+		return nil
+	}
+	usages := make([]db.IUsage, 0)
+	for i := range images {
+		ui := images[i].GetUsages()
+		if len(ui) > 0 {
+			usages = append(usages, ui...)
+		}
+	}
+	return usages
+}

pkg/image/models/image_guest_joint.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
@@ -102,7 +103,7 @@ func (gm *SGuestImageJointManager) GetImagesByFilter(guestImageId string,
 	q = filter(q)
 	images := make([]SImage, 0, len(imageIds))
 	err = db.FetchModelObjects(ImageManager, q, &images)
-	if err != nil {
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
 		return nil, errors.Wrap(err, "fetch images failed")
 	}
 	return images, nil

pkg/image/models/images.go

@@ -806,9 +806,12 @@ func (self *SImage) AllowPerformCancelDelete(ctx context.Context, userCred mccli
 }
 
 func (self *SImage) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "DoCancelPendingDelete")
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }

pkg/keystone/models/identitybase.go

@@ -133,12 +133,7 @@ func (manager *SIdentityBaseResourceManager) ListItemFilter(
 	// override manager.SDomainizedResourceBaseManager.ListItemFilter()
 	if len(query.ProjectDomains) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.ProjectDomains {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.ProjectDomains)
 		domains := DomainManager.Query().SubQuery()
 		subq := domains.Query(domains.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(domains.Field("id"), idList),

pkg/logger/policy/defaults.go

@@ -21,15 +21,16 @@ import (
 )
 
 const (
-	PolicyActionGet  = common_policy.PolicyActionGet
-	PolicyActionList = common_policy.PolicyActionList
+	PolicyActionGet    = common_policy.PolicyActionGet
+	PolicyActionList   = common_policy.PolicyActionList
+	PolicyActionCreate = common_policy.PolicyActionCreate
 )
 
 var (
 	predefinedDefaultPolicies = []rbacutils.SRbacPolicy{
 		{
 			Auth:  true,
-			Scope: rbacutils.ScopeProject,
+			Scope: rbacutils.ScopeUser,
 			Rules: []rbacutils.SRbacRule{
 				{
 					Service:  api.SERVICE_TYPE,
@@ -43,6 +44,12 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "actions",
+					Action:   PolicyActionCreate,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 	}

pkg/mcclient/modules/mod_images.go

@@ -93,6 +93,12 @@ func (this *ImageManager) GetByName(session *mcclient.ClientSession, id string,
 }
 
 func (this *ImageManager) Get(session *mcclient.ClientSession, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	// hack: some GetPropertiesMethod must use HTTP GET action like:
+	// - GET /images/distinct-field
+	// hard code this id currently, should found a better solution
+	if ok, _ := utils.InStringArray(id, []string{"distinct-field"}); ok {
+		return this.ResourceManager.Get(session, id, params)
+	}
 	r, e := this.GetById(session, id, params)
 	if e == nil {
 		return r, e

pkg/monitor/models/datasource.go

@@ -30,7 +30,6 @@ import (
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/wait"
 
-	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -87,25 +86,32 @@ func (man *SDataSourceManager) Run(ctx context.Context) error {
 
 func (man *SDataSourceManager) initDefaultDataSource(ctx context.Context) error {
 	region := options.Options.Region
+	epType := options.Options.SessionEndpointType
 	initF := func() {
 		ds, err := man.GetDefaultSource()
 		if err != nil && err != ErrDataSourceDefaultNotFound {
 			log.Errorf("Get default datasource: %v", err)
 			return
 		}
-		if ds != nil {
-			return
-		}
-		s := auth.GetAdminSessionWithPublic(ctx, region, "")
+		s := auth.GetAdminSession(ctx, region, "")
 		if s == nil {
 			log.Errorf("get empty public session for region %s", region)
 			return
 		}
-		url, err := s.GetServiceURL("influxdb", identityapi.EndpointInterfacePublic)
+		url, err := s.GetServiceURL("influxdb", epType)
 		if err != nil {
 			log.Errorf("get influxdb public url: %v", err)
 			return
 		}
+		if ds != nil {
+			if _, err := db.Update(ds, func() error {
+				ds.Url = url
+				return nil
+			}); err != nil {
+				log.Errorf("update datasource url error: %v", err)
+			}
+			return
+		}
 		ds = &SDataSource{
 			Type: monitor.DataSourceTypeInfluxdb,
 			Url:  url,

pkg/monitor/service/handlers.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"strconv"
 
 	"github.com/gorilla/mux"
@@ -112,4 +113,13 @@ func addMiscHandlers(root *mux.Router) {
 		}
 	}
 	root.HandleFunc("/subscriptions/write", adapterF(performHandler))
+
+	// ref: pkg/appsrv/appsrv:addDefaultHandlers
+	root.HandleFunc("/version", adapterF(appsrv.VersionHandler))
+	root.HandleFunc("/stats", adapterF(appsrv.StatisticHandler))
+	root.HandleFunc("/ping", adapterF(appsrv.PingHandler))
+	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
+
+	// pprof handler
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }

pkg/multicloud/esxi/devtools.go

@@ -173,18 +173,23 @@ func NewVNICDev(host *SHost, mac, driver string, bridge string, vlanId int32, ke
 	var backing types.BaseVirtualDeviceBackingInfo
 	switch inet.(type) {
 	case *SDistributedVirtualPortgroup:
+		// net := inet.(*SDistributedVirtualPortgroup)
+		// port, err := net.FindPort()
+		//if err != nil {
+		//		return nil, errors.Wrap(err, "net.FindPort")
+		//	}
+		//	if port == nil {
+		//		return nil, fmt.Errorf("no active port for dvportgroup %q", net.GetName())
+		//	}
 		net := inet.(*SDistributedVirtualPortgroup)
-		port, err := net.FindPort()
+		dvpg := net.getMODVPortgroup()
+		uuid, err := net.GetDVSUuid()
 		if err != nil {
-			return nil, errors.Wrap(err, "net.FindPort")
-		}
-		if port == nil {
-			return nil, errors.Error("no valid port on DVS, exhausted")
+			return nil, errors.Wrap(err, "GetDVSUuid")
 		}
 		portCon := types.DistributedVirtualSwitchPortConnection{
-			PortgroupKey: port.PortgroupKey,
-			SwitchUuid:   port.DvsUuid,
-			PortKey:      port.Key,
+			PortgroupKey: dvpg.Key,
+			SwitchUuid:   uuid,
 		}
 		backing = &types.VirtualEthernetCardDistributedVirtualPortBackingInfo{Port: portCon}
 	case *SNetwork:

pkg/multicloud/esxi/host.go

@@ -941,6 +941,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 		}
 	}
 
+	var rootDiskSizeMb int64
 	if len(params.Disks) > 0 {
 		driver := params.Disks[0].Driver
 		if driver == "scsi" || driver == "pvscsi" {
@@ -968,29 +969,9 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 			}
 		}
 
-		// resize system disk
-		sysDiskSize := params.Disks[0].Size
-		if sysDiskSize == 0 {
-			sysDiskSize = 30 * 1024
-		}
-		if int64(from.vdisks[0].GetDiskSizeMB()) != sysDiskSize {
-			vdisk := from.vdisks[0].getVirtualDisk()
-			vdisk.CapacityInKB = sysDiskSize * 1024
-			spec := &types.VirtualDeviceConfigSpec{}
-			spec.Operation = types.VirtualDeviceConfigSpecOperationEdit
-			spec.Device = vdisk
-			deviceChange = append(deviceChange, spec)
-			log.Infof("resize system disk: %dGB => %dGB", from.vdisks[0].GetDiskSizeMB()/1024, vdisk.CapacityInKB/1024/1024)
-		}
-		// remove extra disk
-		for i := 1; i < len(from.vdisks); i++ {
-			dev := from.vdisks[i].dev
-			spec := &types.VirtualDeviceConfigSpec{}
-			spec.Operation = types.VirtualDeviceConfigSpecOperationRemove
-			spec.Device = dev
-			spec.FileOperation = types.VirtualDeviceConfigSpecFileOperationDestroy
-			deviceChange = append(deviceChange, spec)
-			log.Debugf("remove disk, index: %d", i)
+		rootDiskSizeMb = params.Disks[0].Size
+		if rootDiskSizeMb == 0 {
+			rootDiskSizeMb = 30 * 1024
 		}
 	}
 
@@ -1056,6 +1037,13 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 	if vm == nil {
 		return nil, errors.Error("clone successfully but unable to NewVirtualMachine")
 	}
+	// resize system disk
+	if rootDiskSizeMb > 0 && int64(vm.vdisks[0].GetDiskSizeMB()) != rootDiskSizeMb {
+		err = vm.vdisks[0].Resize(ctx, rootDiskSizeMb)
+		if err != nil {
+			return vm, errors.Wrap(err, "resize for root disk")
+		}
+	}
 	// add data disk
 	for i := 1; i < len(params.Disks); i++ {
 		size := params.Disks[i].Size

pkg/multicloud/esxi/network.go

@@ -44,7 +44,7 @@ const (
 )
 
 var NETWORK_PROPS = []string{"name", "parent", "summary", "host", "vm"}
-var DVPORTGROUP_PROPS = []string{"name", "parent", "summary", "host", "vm", "config"}
+var DVPORTGROUP_PROPS = []string{"name", "parent", "summary", "host", "vm", "config", "key"}
 
 type SNetwork struct {
 	SManagedObject
@@ -185,6 +185,16 @@ func (net *SDistributedVirtualPortgroup) Uplink() bool {
 	return *dvpg.Config.Uplink
 }
 
+func (net *SDistributedVirtualPortgroup) GetDVSUuid() (string, error) {
+	dvgp := net.getMODVPortgroup()
+	var dvs mo.DistributedVirtualSwitch
+	err := net.manager.reference2Object(*dvgp.Config.DistributedVirtualSwitch, []string{"uuid"}, &dvs)
+	if err != nil {
+		return "", errors.Wrap(err, "reference2Object")
+	}
+	return dvs.Uuid, nil
+}
+
 func (net *SDistributedVirtualPortgroup) FindPort() (*types.DistributedVirtualPort, error) {
 	dvgp := net.getMODVPortgroup()
 	odvs := object.NewDistributedVirtualSwitch(net.manager.client.Client, *dvgp.Config.DistributedVirtualSwitch)

pkg/multicloud/esxi/virtualmachine.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/vmware/govmomi/nfc"
 	"github.com/vmware/govmomi/object"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
@@ -728,8 +729,13 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		return fmt.Errorf("invalid vm")
 	}
 
-	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
-		dev := moVM.Config.Hardware.Device[i]
+	// sort devices via their Key
+	devices := moVM.Config.Hardware.Device
+	sort.Slice(devices, func(i, j int) bool {
+		return devices[i].GetVirtualDevice().Key < devices[j].GetVirtualDevice().Key
+	})
+	for i := 0; i < len(devices); i += 1 {
+		dev := devices[i]
 		devType := reflect.Indirect(reflect.ValueOf(dev)).Type()
 
 		etherType := reflect.TypeOf((*types.VirtualEthernetCard)(nil)).Elem()
@@ -740,7 +746,7 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		if reflectutils.StructContains(devType, etherType) {
 			self.vnics = append(self.vnics, NewVirtualNIC(self, dev, len(self.vnics)))
 		} else if reflectutils.StructContains(devType, diskType) {
-			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vnics)))
+			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vdisks)))
 		} else if reflectutils.StructContains(devType, vgaType) {
 			self.vga = NewVirtualVGA(self, dev, 0)
 		} else if reflectutils.StructContains(devType, cdromType) {
@@ -1169,8 +1175,19 @@ func (self *SVirtualMachine) ExportTemplate(ctx context.Context, idx int, diskPa
 	lr := newLeaseLogger("download vmdk", 5)
 	lr.Log()
 	defer lr.End()
+
+	// filter vmdk item
+	vmdkItems := make([]nfc.FileItem, 0, len(info.Items)/2)
+	for i := range info.Items {
+		if strings.HasSuffix(info.Items[i].Path, ".vmdk") {
+			vmdkItems = append(vmdkItems, info.Items[i])
+		} else {
+			log.Infof("item.Path does not end in '.vmdk': %#v", info.Items[i])
+		}
+	}
+
 	log.Debugf("download to %s start...", diskPath)
-	err = lease.DownloadFile(ctx, diskPath, info.Items[idx], soap.Download{Progress: lr})
+	err = lease.DownloadFile(ctx, diskPath, vmdkItems[idx], soap.Download{Progress: lr})
 	if err != nil {
 		return errors.Wrap(err, "lease.DownloadFile")
 	}

pkg/multicloud/google/instance.go

@@ -529,7 +529,7 @@ func getDiskInfo(disk string) (cloudprovider.SDiskInfo, error) {
 	result := cloudprovider.SDiskInfo{}
 	diskInfo := strings.Split(disk, ":")
 	for _, d := range diskInfo {
-		if utils.IsInStringArray(d, []string{api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_LOCAL_SSD}) {
+		if utils.IsInStringArray(d, []string{api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_LOCAL_SSD, api.STORAGE_GOOGLE_PD_BALANCED}) {
 			result.StorageType = d
 		} else if memSize, err := fileutils.GetSizeMb(d, 'M', 1024); err == nil {
 			result.SizeGB = memSize >> 10

pkg/multicloud/huawei/client/modules/mod_disks.go

@@ -33,7 +33,7 @@ func NewDiskManager(regionId string, projectId string, signer auth.Signer, debug
 		Keyword:       "volume",
 		KeywordPlural: "volumes",
 
-		ResourceKeyword: "volumes",
+		ResourceKeyword: "cloudvolumes",
 	}}
 }
 

pkg/multicloud/qcloud/monitor.go

@@ -59,6 +59,7 @@ func (r *SRegion) metricsRequest(action string, params map[string]string) (jsonu
 func (r *SRegion) GetMonitorData(name string, ns string, since time.Time, until time.Time,
 	demensions []SQcMetricDimension) ([]SDataPoint, error) {
 	params := make(map[string]string)
+	params["Region"] = r.Region
 	params["MetricName"] = name
 	params["Namespace"] = ns
 	if !since.IsZero() {

pkg/multicloud/qcloud/project.go

@@ -79,6 +79,9 @@ func (client *SQcloudClient) GetProjects() ([]SProject, error) {
 	projects := []SProject{}
 	params := map[string]string{"allList": "1"}
 	resp, err := client.accountRequestRequest("DescribeProject", params)
+	if err != nil {
+		return nil, errors.Wrapf(err, "DescribeProject")
+	}
 	err = resp.Unmarshal(&projects)
 	if err != nil {
 		return nil, errors.Wrap(err, "resp.Unmarshal")

pkg/util/logclient/logclient.go

@@ -159,11 +159,12 @@ func addLog(model IObject, action string, iNotes interface{}, userCred mcclient.
 	}
 
 	logentry.Add(jsonutils.NewString(notes), "notes")
+
 	logclientWorkerMan.Run(func() {
 		s := DefaultSessionGenerator(context.Background(), userCred, "", "")
 		_, err := api.Create(s, logentry)
 		if err != nil {
-			log.Errorf("create action log failed %s", err)
+			log.Errorf("create action log %s failed %s", logentry, err)
 		}
 	}, nil, nil)
 }

pkg/util/stringutils2/i18n.go

@@ -23,6 +23,16 @@ func IsUtf8(str string) bool {
 	return false
 }
 
+func RemoveUtf8Strings(idOrNames []string) []string {
+	ids := make([]string, 0)
+	for _, idOrName := range idOrNames {
+		if !IsUtf8(idOrName) {
+			ids = append(ids, idOrName)
+		}
+	}
+	return ids
+}
+
 func IsPrintableAscii(b byte) bool {
 	if b >= 32 && b <= 126 {
 		return true

pkg/util/stringutils2/i18n_test.go

@@ -15,6 +15,7 @@
 package stringutils2
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -34,6 +35,31 @@ func TestIsUtf8(t *testing.T) {
 	}
 }
 
+func TestRemoveUtf8Strings(t *testing.T) {
+	cases := []struct {
+		in   []string
+		want []string
+	}{
+		{
+			in:   []string{},
+			want: []string{},
+		},
+		{
+			in:   []string{"en", "中文"},
+			want: []string{"en"},
+		},
+		{
+			in:   []string{"中文"},
+			want: []string{},
+		},
+	}
+	for _, c := range cases {
+		if got := RemoveUtf8Strings(c.in); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("RemoveUtf8Strings %s got %v want %v", c.in, got, c.want)
+		}
+	}
+}
+
 func TestIsPrintableAscii(t *testing.T) {
 	cases := []struct {
 		in   string

pkg/webconsole/service/service.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
-	"net/http/pprof"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"strconv"
@@ -133,9 +133,5 @@ func addMiscHandlers(root *mux.Router) {
 	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
 
 	// pprof handler
-	root.HandleFunc("/debug/pprof/", pprof.Index)
-	root.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-	root.HandleFunc("/debug/pprof/profile", pprof.Profile)
-	root.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-	root.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }

vendor/modules.txt

@@ -1007,7 +1007,7 @@ sigs.k8s.io/yaml
 yunion.io/x/executor/apis
 yunion.io/x/executor/client
 yunion.io/x/executor/server
-# yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8
+# yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
 yunion.io/x/jsonutils
 # yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 yunion.io/x/log
@@ -1049,7 +1049,7 @@ yunion.io/x/pkg/util/workqueue
 yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6
+# yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
 yunion.io/x/sqlchemy
 # yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 yunion.io/x/structarg

vendor/yunion.io/x/jsonutils/access.go

@@ -53,8 +53,17 @@ func NewInt(val int64) *JSONInt {
 	return &JSONInt{data: val}
 }
 
+//deprecated
 func NewFloat(val float64) *JSONFloat {
-	return &JSONFloat{data: val}
+	return &JSONFloat{data: val, bit: 64}
+}
+
+func NewFloat64(val float64) *JSONFloat {
+	return &JSONFloat{data: val, bit: 64}
+}
+
+func NewFloat32(val float32) *JSONFloat {
+	return &JSONFloat{data: float64(val), bit: 32}
 }
 
 func NewBool(val bool) *JSONBool {

vendor/yunion.io/x/jsonutils/jsonutils.go

@@ -91,6 +91,7 @@ type JSONInt struct {
 type JSONFloat struct {
 	JSONValue
 	data float64
+	bit  int
 }
 
 type JSONBool struct {

vendor/yunion.io/x/jsonutils/marshal.go

@@ -150,13 +150,23 @@ func marshalInt64(val int64, info *reflectutils.SStructFieldInfo) JSONObject {
 	}
 }
 
-func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo) JSONObject {
+func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo, bit int) JSONObject {
 	if val == 0.0 && info != nil && info.OmitZero {
 		return JSONNull
 	} else if info != nil && info.ForceString {
 		return NewString(fmt.Sprintf("%f", val))
 	} else {
-		return NewFloat(val)
+		return NewFloat64(val)
+	}
+}
+
+func marshalFloat32(val float32, info *reflectutils.SStructFieldInfo, bit int) JSONObject {
+	if val == 0.0 && info != nil && info.OmitZero {
+		return JSONNull
+	} else if info != nil && info.ForceString {
+		return NewString(fmt.Sprintf("%f", val))
+	} else {
+		return NewFloat32(val)
 	}
 }
 
@@ -312,9 +322,12 @@ func marshalValue(objValue reflect.Value, info *reflectutils.SStructFieldInfo) J
 		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
 		intValue := objValue.Convert(gotypes.Int64Type)
 		return marshalInt64(intValue.Interface().(int64), info)
-	case reflect.Float32, reflect.Float64:
-		floatValue := objValue.Convert(gotypes.Float64Type)
-		return marshalFloat64(floatValue.Interface().(float64), info)
+	case reflect.Float32:
+		floatVal := objValue.Convert(gotypes.Float32Type)
+		return marshalFloat32(floatVal.Interface().(float32), info, 32)
+	case reflect.Float64:
+		floatVal := objValue.Convert(gotypes.Float64Type)
+		return marshalFloat64(floatVal.Interface().(float64), info, 64)
 	case reflect.Interface, reflect.Ptr:
 		if objValue.IsNil() {
 			return JSONNull

vendor/yunion.io/x/jsonutils/string.go

@@ -19,7 +19,10 @@ func (this *JSONInt) String() string {
 }
 
 func (this *JSONFloat) String() string {
-	return strconv.FormatFloat(this.data, 'g', -1, 64)
+	if this.bit != 32 && this.bit != 64 {
+		this.bit = 64
+	}
+	return strconv.FormatFloat(this.data, 'f', -1, this.bit)
 }
 
 func (this *JSONBool) String() string {

vendor/yunion.io/x/jsonutils/unmarshal.go

@@ -524,6 +524,9 @@ func (this *JSONDict) unmarshalMap(val reflect.Value) error {
 		k, vinf := iter.Get()
 		v := vinf.(JSONObject)
 		keyVal := reflect.ValueOf(k)
+		if keyType != keyVal.Type() {
+			keyVal = keyVal.Convert(keyType)
+		}
 		valVal := reflect.New(valType.Elem()).Elem()
 
 		err := v.unmarshalValue(valVal)

vendor/yunion.io/x/sqlchemy/conditions.go

@@ -378,6 +378,14 @@ func Like(f IQueryField, v string) ICondition {
 	return &c
 }
 
+func ContainsAny(f IQueryField, v []string) ICondition {
+	conds := make([]ICondition, len(v))
+	for i := range v {
+		conds[i] = Contains(f, v[i])
+	}
+	return OR(conds...)
+}
+
 func Contains(f IQueryField, v string) ICondition {
 	v = likeEscape(v)
 	nv := fmt.Sprintf("%%%s%%", v)

vendor/yunion.io/x/sqlchemy/const.go

@@ -14,6 +14,11 @@
 
 package sqlchemy
 
+import (
+	"strconv"
+	"strings"
+)
+
 const (
 	SQL_OP_AND      = "AND"
 	SQL_OP_OR       = "OR"
@@ -48,3 +53,26 @@ const (
 	TAG_CREATE_TIMESTAMP = "created_at"
 	TAG_ALLOW_ZERO       = "allow_zero"
 )
+
+var (
+	INT_WIDTH_DEFAULT = map[string]int{
+		"TINYINT":  4,
+		"SMALLINT": 6,
+		"INT":      11,
+		"BIGINT":   20,
+	}
+	UNSIGNED_INT_WIDTH_DEFAULT = map[string]int{
+		"TINYINT":  3,
+		"SMALLINT": 5,
+		"INT":      10,
+		"BIGINT":   20,
+	}
+)
+
+func intWidthString(typeStr string) string {
+	return strconv.FormatInt(int64(INT_WIDTH_DEFAULT[strings.ToUpper(typeStr)]), 10)
+}
+
+func uintWidthString(typeStr string) string {
+	return strconv.FormatInt(int64(UNSIGNED_INT_WIDTH_DEFAULT[strings.ToUpper(typeStr)]), 10)
+}

vendor/yunion.io/x/sqlchemy/inc.go

@@ -47,6 +47,11 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i
 
 	dataValue := reflect.Indirect(reflect.ValueOf(diff))
 	fields := reflectutils.FetchStructFieldValueSet(dataValue)
+	var targetFields reflectutils.SStructFieldValueSet
+	if target != nil {
+		targetValue := reflect.Indirect(reflect.ValueOf(target))
+		targetFields = reflectutils.FetchStructFieldValueSet(targetValue)
+	}
 
 	primaries := make(map[string]interface{})
 	vars := make([]interface{}, 0)
@@ -58,6 +63,9 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i
 		k := c.Name()
 		v, _ := fields.GetInterface(k)
 		if c.IsPrimary() {
+			if targetFields != nil {
+				v, _ = targetFields.GetInterface(k)
+			}
 			if !gotypes.IsNil(v) && !c.IsZero(v) {
 				primaries[k] = v
 			} else if c.IsText() {

vendor/yunion.io/x/sqlchemy/parser.go

@@ -55,35 +55,35 @@ func getFiledTypeCol(fieldType reflect.Type, fieldname string, tagmap map[string
 		col := NewTextColumn(fieldname, tagmap, isPointer)
 		return &col
 	case reflect.Int, reflect.Int32:
-		tagmap[TAG_WIDTH] = "11"
+		tagmap[TAG_WIDTH] = intWidthString("INT")
 		col := NewIntegerColumn(fieldname, "INT", false, tagmap, isPointer)
 		return &col
 	case reflect.Int8:
-		tagmap[TAG_WIDTH] = "4"
+		tagmap[TAG_WIDTH] = intWidthString("TINYINT")
 		col := NewIntegerColumn(fieldname, "TINYINT", false, tagmap, isPointer)
 		return &col
 	case reflect.Int16:
-		tagmap[TAG_WIDTH] = "6"
+		tagmap[TAG_WIDTH] = intWidthString("SMALLINT")
 		col := NewIntegerColumn(fieldname, "SMALLINT", false, tagmap, isPointer)
 		return &col
 	case reflect.Int64:
-		tagmap[TAG_WIDTH] = "20"
+		tagmap[TAG_WIDTH] = intWidthString("BIGINT")
 		col := NewIntegerColumn(fieldname, "BIGINT", false, tagmap, isPointer)
 		return &col
 	case reflect.Uint, reflect.Uint32:
-		tagmap[TAG_WIDTH] = "11"
+		tagmap[TAG_WIDTH] = uintWidthString("INT")
 		col := NewIntegerColumn(fieldname, "INT", true, tagmap, isPointer)
 		return &col
 	case reflect.Uint8:
-		tagmap[TAG_WIDTH] = "4"
+		tagmap[TAG_WIDTH] = uintWidthString("TINYINT")
 		col := NewIntegerColumn(fieldname, "TINYINT", true, tagmap, isPointer)
 		return &col
 	case reflect.Uint16:
-		tagmap[TAG_WIDTH] = "6"
+		tagmap[TAG_WIDTH] = uintWidthString("SMALLINT")
 		col := NewIntegerColumn(fieldname, "SMALLINT", true, tagmap, isPointer)
 		return &col
 	case reflect.Uint64:
-		tagmap[TAG_WIDTH] = "20"
+		tagmap[TAG_WIDTH] = uintWidthString("BIGINT")
 		col := NewIntegerColumn(fieldname, "BIGINT", true, tagmap, isPointer)
 		return &col
 	case reflect.Bool:

vendor/yunion.io/x/sqlchemy/sync.go

@@ -45,7 +45,8 @@ func decodeSqlTypeString(typeStr string) []string {
 	if len(matches) >= 3 {
 		return matches[1:]
 	} else {
-		return []string{typeStr}
+		parts := strings.Split(typeStr, " ")
+		return []string{parts[0]}
 	}
 }
 
@@ -98,6 +99,13 @@ func (info *SSqlColumnInfo) toColumnSpec() IColumnSpec {
 		if strings.HasSuffix(info.Type, " unsigned") {
 			unsigned = true
 		}
+		if _, ok := tagmap[TAG_WIDTH]; !ok {
+			if unsigned {
+				tagmap[TAG_WIDTH] = uintWidthString(typeStr)
+			} else {
+				tagmap[TAG_WIDTH] = intWidthString(typeStr)
+			}
+		}
 		c := NewIntegerColumn(info.Field, typeStr, unsigned, tagmap, false)
 		return &c
 	} else if typeStr == "FLOAT" || typeStr == "DOUBLE" {
@@ -408,12 +416,13 @@ func (ts *STableSpec) Sync() error {
 	return nil
 }
 
-func (ts *STableSpec) CheckSync() {
+func (ts *STableSpec) CheckSync() error {
 	sqls := ts.SyncSQL()
 	if len(sqls) > 0 {
 		for _, sql := range sqls {
 			fmt.Println(sql)
 		}
-		log.Fatalf("DB table %q not in sync", ts.name)
+		return fmt.Errorf("DB table %q not in sync", ts.name)
 	}
+	return nil
 }

vendor/yunion.io/x/sqlchemy/union.go

@@ -146,7 +146,16 @@ func (uq *SUnion) Variables() []interface{} {
 	return ret
 }
 
+// deprecated
 func Union(query ...IQuery) *SUnion {
+	u, err := UnionWithError(query...)
+	if err != nil {
+		log.Fatalf("Fatal: %s", err.Error())
+	}
+	return u
+}
+
+func UnionWithError(query ...IQuery) (*SUnion, error) {
 	fieldNames := make([]string, 0)
 	for _, f := range query[0].QueryFields() {
 		fieldNames = append(fieldNames, f.Name())
@@ -155,11 +164,11 @@ func Union(query ...IQuery) *SUnion {
 	for i := 1; i < len(query); i += 1 {
 		qfields := query[i].QueryFields()
 		if len(fieldNames) != len(qfields) {
-			log.Fatalf("cannot union, number of fields not match!")
+			return nil, fmt.Errorf("cannot union, number of fields not match!")
 		}
 		for i := range qfields {
 			if fieldNames[i] != qfields[i].Name() {
-				log.Fatalf("cannot union, name of fields not match!")
+				return nil, fmt.Errorf("cannot union, name of fields not match!")
 			}
 		}
 	}
@@ -176,7 +185,7 @@ func Union(query ...IQuery) *SUnion {
 		fields[i] = &SUnionQueryField{name: fieldNames[i], union: uq}
 	}
 
-	return uq
+	return uq, nil
 }
 
 func (uq *SUnion) Query(f ...IQueryField) *SQuery {

vendor/yunion.io/x/sqlchemy/update.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"strings"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
@@ -74,14 +75,21 @@ func (ud *SUpdateDiff) String() string {
 		utils.TruncateString(ud.new, 32))
 }
 
+func (ud SUpdateDiff) jsonObj() jsonutils.JSONObject {
+	r := jsonutils.NewDict()
+	r.Set("old", jsonutils.Marshal(ud.old))
+	r.Set("new", jsonutils.Marshal(ud.new))
+	return r
+}
+
 type UpdateDiffs map[string]SUpdateDiff
 
 func (uds UpdateDiffs) String() string {
-	items := make([]string, 0, len(uds))
-	for k, v := range uds {
-		items = append(items, fmt.Sprintf("%s:%s", k, v.String()))
+	obj := jsonutils.NewDict()
+	for k := range uds {
+		obj.Set(k, uds[k].jsonObj())
 	}
-	return strings.Join(items, ";")
+	return obj.String()
 }
 
 func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) {