Merge pull request #11410 from yousong/automated-cherry-pick-of-#11406-upstream-release-3.4

Automated cherry pick of #11406: webconsole: wait zombies

.gitignore

@@ -31,6 +31,7 @@ _output
 /structargtest
 .idea
 .vscode
+intellij.yaml
 GPATH
 GRTAGS
 GTAGS

Makefile

@@ -9,7 +9,7 @@ BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
 
 ifeq ($(ONECLOUD_CI_BUILD),)
 	GIT_COMMIT := $(shell git rev-parse --short HEAD)
-	GIT_BRANCH := $(shell git name-rev --name-only HEAD)
+	GIT_BRANCH := $(shell git branch -r --contains | head -1 | sed -E -e "s%(HEAD ->|origin|upstream)/?%%g" | xargs)
 	GIT_VERSION := $(shell git describe --always --tags --abbrev=14 $(GIT_COMMIT)^{commit})
 	GIT_TREE_STATE := $(shell s=`git status --porcelain 2>/dev/null`; if [ -z "$$s" ]; then echo "clean"; else echo "dirty"; fi)
 	BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT

@@ -2,7 +2,7 @@
 策略名称: {{.name}}
 触发时间: {{.start_time}}
 报警级别: {{.level}}
-触发条件: {{.description}}
+触发条件: {{.description | unescaped}}
 资源数量：{{len .matches}}
 资源名称：{{.resource_name}}
 

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT

@@ -2,7 +2,7 @@
 AlertName: {{.name}}
 Time: {{.start_time}}
 Level: {{.level}}
-TriggerCondition: {{html .description}}
+TriggerCondition: {{.description | unescaped}}
 ResourceCount: {{len .matches}}
 ResourceName: {{.resource_name}}
 

cmd/ansibleserver/main.go

@@ -15,9 +15,14 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/ansibleserver/service"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/baremetal-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/baremetal/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/climc/shell/compute/cloudregions.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -26,6 +27,23 @@ import (
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.Cloudregions).WithKeyword("cloud-region")
+	cmd.PerformClass("sync-skus", &options.CloudregionSkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "cloud-region-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.Cloudregions.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudregionListOptions struct {
 		options.BaseListOptions
 

cmd/climc/shell/compute/dbinstance_skus.go

@@ -15,12 +15,30 @@
 package compute
 
 import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.DBInstanceSkus).WithKeyword("dbinstance-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "dbinstance-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.DBInstanceSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type DBInstanceSkuListOption struct {
 		options.BaseListOptions
 		Engine        string
@@ -54,5 +72,4 @@ func init() {
 		printObject(result)
 		return nil
 	})
-
 }

cmd/climc/shell/compute/elasticcache.go

@@ -19,12 +19,30 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ElasticcacheSkus).WithKeyword("elastic-cache-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "elastic-cache-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ElasticcacheSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.BaseListOptions{}, "elastic-cache-list", "List elastisc cache instance", func(s *mcclient.ClientSession, opts *options.BaseListOptions) error {
 		params, err := options.ListStructToParams(opts)
 		if err != nil {

cmd/climc/shell/compute/elasticips.go

@@ -50,7 +50,7 @@ func init() {
 		Manager    string `help:"cloud provider"`
 		Region     string `help:"cloud region in which EIP is allocated"`
 		Bandwidth  int    `help:"Bandwidth in Mbps"`
-		Ip         string `help:"IP address of the EIP"`
+		IpAddr     string `help:"IP address of the EIP" json:"ip_addr"`
 		Network    string `help:"Network of the EIP"`
 		ChargeType string `help:"bandwidth charge type" choices:"traffic|bandwidth"`
 	}
@@ -75,8 +75,8 @@ func init() {
 			params.Add(jsonutils.NewString(args.Network), "network")
 		}
 
-		if len(args.Ip) > 0 {
-			params.Add(jsonutils.NewString(args.Ip), "ip")
+		if len(args.IpAddr) > 0 {
+			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
 
 		result, err := modules.Elasticips.Create(s, params)

cmd/climc/shell/compute/hosts.go

@@ -407,12 +407,14 @@ func init() {
 	})
 
 	type HostAddNetIfOptions struct {
-		ID     string `help:"ID or Name of host"`
-		WIRE   string `help:"ID or Name of wire to attach"`
-		MAC    string `help:"Mac address of NIC"`
-		INDEX  int64  `help:"nic index"`
-		Type   string `help:"Nic type" choices:"admin|ipmi"`
-		IpAddr string `help:"IP address"`
+		ID        string `help:"ID or Name of host"`
+		WIRE      string `help:"ID or Name of wire to attach"`
+		MAC       string `help:"Mac address of NIC"`
+		INDEX     int64  `help:"nic index"`
+		Type      string `help:"Nic type" choices:"admin|ipmi"`
+		IpAddr    string `help:"IP address"`
+		Bridge    string `help:"Bridge of hostwire"`
+		Interface string `help:"Interface name, eg:eth0, en0"`
 	}
 	R(&HostAddNetIfOptions{}, "host-add-netif", "Host add a NIC", func(s *mcclient.ClientSession, args *HostAddNetIfOptions) error {
 		params := jsonutils.NewDict()
@@ -426,6 +428,12 @@ func init() {
 		if len(args.IpAddr) > 0 {
 			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
+		if len(args.Bridge) > 0 {
+			params.Add(jsonutils.NewString(args.Bridge), "bridge")
+		}
+		if len(args.Interface) > 0 {
+			params.Add(jsonutils.NewString(args.Interface), "interface")
+		}
 		result, err := modules.Hosts.PerformAction(s, args.ID, "add-netif", params)
 		if err != nil {
 			return err

cmd/climc/shell/compute/quotas.go

@@ -316,7 +316,7 @@ func init() {
 		QuotaSetBaseOptions
 		IdentityQuotaOptions
 	}
-	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *InfrasQuotaSetOptions) error {
+	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *IdentityQuotaSetOptions) error {
 		params := jsonutils.Marshal(args)
 		quotas, e := modules.IdentityQuotas.DoQuotaSet(s, params)
 		if e != nil {

cmd/climc/shell/compute/schedulers.go

@@ -123,9 +123,10 @@ func init() {
 		})
 
 	type SchedulerHistoryListOptions struct {
-		Limit  int  `default:"50" help:"Page limit"`
-		Offset int  `default:"0" help:"Page offset"`
-		All    bool `help:"Show all histories, including scheduler-test"`
+		Limit        int  `default:"50" help:"Page limit"`
+		Offset       int  `default:"0" help:"Page offset"`
+		All          bool `help:"Show all histories, including scheduler-test"`
+		IsSuggestion bool `help:"Only show forcast suggestion history"`
 	}
 	R(&SchedulerHistoryListOptions{}, "scheduler-history-list", "Show scheduler history list",
 		func(s *mcclient.ClientSession, args *SchedulerHistoryListOptions) error {
@@ -138,17 +139,14 @@ func init() {
 			if args.Offset > 0 {
 				params.Add(jsonutils.NewInt(int64(args.Offset)), "offset")
 			}
-			if args.All {
-				params.Add(jsonutils.JSONTrue, "all")
-			} else {
-				params.Add(jsonutils.JSONFalse, "all")
-			}
+			params.Add(jsonutils.NewBool(args.All), "all")
+			params.Add(jsonutils.NewBool(args.IsSuggestion), "is_suggestion")
 			result, err := modules.SchedManager.HistoryList(s, params)
 			if err != nil {
 				return err
 			}
 			printList(modulebase.JSON2ListResult(result), []string{
-				"session_id", "time", "status", "consuming",
+				"session_id", "time", "status", "consuming", "is_suggestion",
 			})
 			return nil
 		})

cmd/climc/shell/compute/skus.go

@@ -17,12 +17,30 @@ package compute
 import (
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ServerSkus).WithKeyword("server-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "server-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ServerSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type ServerSkusListOptions struct {
 		options.BaseListOptions
 		Cloudregion    string  `help:"region Id or name"`

cmd/climc/shell/compute/storages.go

@@ -61,13 +61,14 @@ func init() {
 		ID                    string  `help:"ID or Name of storage to update"`
 		Name                  string  `help:"New Name of storage"`
 		Desc                  string  `help:"Description"`
-		CommitBound           float64 `help:"Upper bound of storage overcommit rate"`
+		CommitBound           float64 `help:"Upper bound of storage overcommit rate" json:"cmtbound"`
 		MediumType            string  `help:"Medium type" choices:"ssd|rotate"`
 		RbdRadosMonOpTimeout  int64   `help:"ceph rados_mon_op_timeout"`
 		RbdRadosOsdOpTimeout  int64   `help:"ceph rados_osd_op_timeout"`
 		RbdClientMountTimeout int64   `help:"ceph client_mount_timeout"`
 		RbdKey                string  `help:"ceph rbd key"`
 		Reserved              string  `help:"Reserved storage space"`
+		Capacity              int     `help:"Capacity for storage"`
 	}
 	R(&StorageUpdateOptions{}, "storage-update", "Update a storage", func(s *mcclient.ClientSession, args *StorageUpdateOptions) error {
 		params, err := options.StructToParams(args)

cmd/climc/shell/compute/webconsole.go

@@ -42,34 +42,24 @@ func init() {
 		if err != nil {
 			return err
 		}
-		connParams, err := obj.GetString("connect_params")
-		if err != nil {
+
+		resp := &webconsole_api.ServerRemoteConsoleResponse{}
+		if err := obj.Unmarshal(resp); err != nil {
 			return err
 		}
-		if decodeStr, err := base64.StdEncoding.DecodeString(connParams); err == nil {
-			connParams = string(decodeStr)
-		}
-		var query url.Values
-		connUrl, err := url.ParseRequestURI(connParams)
-		if err == nil {
-			query = connUrl.Query()
-		} else {
-			query, err = url.ParseQuery(connParams)
-			if err != nil {
-				return err
-			}
-		}
-		protocol := query.Get("protocol")
-		if !utils.IsInStringArray(protocol, []string{
+		connectParams := resp.GetConnectParams()
+		if protocol, err := resp.GetConnectProtocol(); err != nil {
+			return err
+		} else if !utils.IsInStringArray(protocol, []string{
 			command.PROTOCOL_TTY, webconsole_api.VNC,
 			webconsole_api.SPICE, webconsole_api.WMKS,
 		}) {
-			fmt.Println(connParams)
+			fmt.Println(connectParams)
 			return nil
 		}
 
-		newQuery := url.Values(make(map[string][]string))
-		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connParams)))
+		newQuery := url.Values{}
+		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connectParams)))
 		u.RawQuery = newQuery.Encode()
 		fmt.Println(u.String())
 		return nil

cmd/climc/shell/helper.go

@@ -284,7 +284,10 @@ type IPerformOpt interface {
 func (cmd ResourceCmd) Perform(action string, args IPerformOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IPerformOpt) error {
-		params := jsonutils.Marshal(args) // .Params()
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
 		ret, err := man.(modulebase.Manager).PerformAction(s, args.GetId(), action, params)
 		if err != nil {
 			return err

cmd/climc/shell/identity/policies.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -183,6 +184,7 @@ func init() {
 	type PolicyShowOptions struct {
 		ID     string `help:"ID of policy"`
 		Format string `help:"policy format, default yaml" default:"yaml" choices:"yaml|json"`
+		Save   string `help:"save policy data into a file"`
 	}
 	R(&PolicyShowOptions{}, "policy-show", "Show policy", func(s *mcclient.ClientSession, args *PolicyShowOptions) error {
 		query := jsonutils.NewDict()
@@ -192,6 +194,15 @@ func init() {
 			return err
 		}
 		printObject(result)
+		if len(args.Save) > 0 {
+			if args.Format == "yaml" {
+				yaml, _ := result.GetString("policy")
+				fileutils2.FilePutContents(args.Save, yaml, false)
+			} else {
+				p, _ := result.Get("policy")
+				fileutils2.FilePutContents(args.Save, p.PrettyString(), false)
+			}
+		}
 		return nil
 	})
 

cmd/climc/shell/misc/copyright.go

@@ -36,6 +36,8 @@ func init() {
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
 		License   *string `help:"the license generator website address"`
+		BrandCn   *string `help:"the brand name in Chinese"`
+		BrandEn   *string `help:"the brand name in English"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -52,6 +54,14 @@ func init() {
 			params.Add(jsonutils.NewString(*args.License), "license")
 		}
 
+		if args.BrandCn != nil {
+			params.Add(jsonutils.NewString(*args.BrandCn), "brand_cn")
+		}
+
+		if args.BrandEn != nil {
+			params.Add(jsonutils.NewString(*args.BrandEn), "brand_en")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

cmd/esxi-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/esxi/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/glance/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/image/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/host-deployer/main.go

@@ -14,9 +14,16 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	deployer := deployserver.NewDeployService()
 	deployer.StartService()
 }

cmd/host/main.go

@@ -15,14 +15,19 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/service"
 	"yunion.io/x/onecloud/pkg/hostman"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	var srv = &hostman.SHostService{}
 	srv.SServiceBase = &service.SServiceBase{
 		Service: srv,

cmd/vpcagent/main.go

@@ -26,6 +26,7 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/vpcagent/options"
 	_ "yunion.io/x/onecloud/pkg/vpcagent/ovn"
 	"yunion.io/x/onecloud/pkg/vpcagent/worker"
@@ -52,8 +53,12 @@ func main() {
 	}
 
 	{
+
+		ctx := context.Background()
+		ctx, cancelFunc := context.WithCancel(ctx)
+		go procutils.WaitZombieLoop(ctx)
+
 		wg := &sync.WaitGroup{}
-		ctx, cancelFunc := context.WithCancel(context.Background())
 		ctx = context.WithValue(ctx, "wg", wg)
 		wg.Add(1)
 		go w.Start(ctx)

cmd/webconsole/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

pkg/apigateway/handler/auth.go

@@ -156,6 +156,9 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 			}
 		}
 		resp.Add(domains, "domains")
+		resp.Add(jsonutils.JSONTrue, "return_full_domains")
+	} else {
+		resp.Add(jsonutils.JSONFalse, "return_full_domains")
 	}
 
 	filters := jsonutils.NewDict()
@@ -962,6 +965,12 @@ func getUserInfo2(s *mcclient.ClientSession, uid string, pid string, loginIp str
 		data.Add(jsonutils.JSONFalse, "non_default_domain_projects")
 	}
 
+	if options.Options.EnableQuotaCheck {
+		data.Add(jsonutils.JSONTrue, "enable_quota_check")
+	} else {
+		data.Add(jsonutils.JSONFalse, "enable_quota_check")
+	}
+
 	data.Add(jsonutils.NewString(getSsoCallbackUrl()), "sso_callback_url")
 
 	return data, nil

pkg/apis/compute/cloudregions_const.go

@@ -114,6 +114,7 @@ const (
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
 	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
+	CITY_PHOENIX        = "Phoenix"        //菲尼克斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -150,7 +151,8 @@ const (
 	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
 
 	// 巴西
-	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+	CITY_SAO_PAULO      = "Sao Paulo"      //圣保罗
+	CITY_RIO_DE_JANEIRO = "Rio de Janeiro" // 里约热内卢
 
 	// 智利
 	CITY_SANTIAGO = "Santiago" // 圣地亚哥

pkg/apis/compute/dbinstance.go

@@ -24,12 +24,13 @@ type DBInstanceCreateInput struct {
 	apis.VirtualResourceCreateInput
 	DeletePreventableCreateInput
 
-	// Ip子网名称或Id,建议使用Id
+	// Ip子网Id
 	// 谷歌云并不实际使用Ip子网,仅仅通过Ip子网确定Vpc
 	// required: true
-	Network string `json:"network"`
+	NetworkId string `json:"network_id"`
+
 	// swagger:ignore
-	NetworkId string
+	Network string `json:"network" yunion-deprecated-by:"network_id"`
 
 	// Ip子网内的地址,不填则按照ip子网的地址分配策略分配一个ip
 	// required: false
@@ -66,8 +67,10 @@ type DBInstanceCreateInput struct {
 	// swagger:ignore
 	CloudregionId string
 
+	VpcId string `json:"vpc_id"`
+
 	// swagger:ignore
-	VpcId string
+	Vpc string `json:"vpc" yunion-deprecated-by:"vpc_id"`
 
 	// swagger:ignore
 	ManagerId string
@@ -142,6 +145,9 @@ type DBInstanceCreateInput struct {
 	// required: true
 	DiskSizeGB int `json:"disk_size_gb"`
 
+	// 指定连接端口
+	Port int `json:"port"`
+
 	// rds初始化密码
 	// 阿里云不需要此参数
 	// 华为云会默认创建一个用户,若不传此参数, 则为随机密码
@@ -163,6 +169,9 @@ type DBInstanceCreateInput struct {
 
 	// swagger:ignore
 	Provider string
+
+	// 从备份中创建新实例
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
 }
 
 type SDBInstanceChangeConfigInput struct {

pkg/apis/compute/elasticip.go

@@ -67,6 +67,8 @@ type SElasticipCreateInput struct {
 	// 子网Id
 	// 私有云创建此参数必传,例如Openstack, ZStack
 	NetworkId string `json:"network_id"`
+	// 指定新建EIP的地址
+	IpAddr string `json:"ip_addr"`
 }
 
 type ElasticipDetails struct {

pkg/apis/compute/elasticips_const.go

@@ -42,7 +42,6 @@ const (
 
 	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
 	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
-	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
 )
 
 var (

pkg/apis/compute/guest_const.go

@@ -249,6 +249,8 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 }
 
 const (
+	VM_DEFAULT_WINDOWS_LOGIN_USER     = "Administrator"
+	VM_DEFAULT_LINUX_LOGIN_USER       = "root"
 	VM_AWS_DEFAULT_LOGIN_USER         = "ec2user"
 	VM_AWS_DEFAULT_WINDOWS_LOGIN_USER = "Administrator"
 	VM_AZURE_DEFAULT_LOGIN_USER       = "toor"

pkg/apis/compute/guests.go

@@ -41,6 +41,11 @@ type ServerListInput struct {
 	Baremetal *bool `json:"baremetal"`
 	// 只列出GPU主机
 	Gpu *bool `json:"gpu"`
+	// 只列出还有备份机的主机
+	Backup *bool `json:"bakcup"`
+	// 列出指定类型的主机
+	// enum: normal,gpu,backup
+	ServerType string `json:"server_type"`
 	// 列出管理安全组为指定安全组的主机
 	AdminSecgroup string `json:"admin_security"`
 	// 列出Hypervisor为指定值的主机
@@ -209,6 +214,8 @@ type ServerDetails struct {
 	Vpc string `json:"vpc"`
 	// 归属VPC ID
 	VpcId string `json:"vpc_id"`
+	// Vpc外网访问模式
+	VpcExternalAccessMode string `json:"vpc_external_access_mode"`
 
 	// 关联安全组列表
 	Secgroups []apis.StandaloneShortDesc `json:"secgroups"`

pkg/apis/compute/network_const.go

@@ -62,6 +62,7 @@ var (
 		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,
+		CLOUD_PROVIDER_OPENSTACK,
 	}
 )
 

pkg/apis/compute/sku_const.go

@@ -155,3 +155,25 @@ type DBInstanceSkuListInput struct {
 	Zone2 []string `json:"zone2"`
 	Zone3 []string `json:"zone3"`
 }
+
+type SkuSyncInput struct {
+	// 云平台名称
+	// example: Google
+	Provider string `json:"provider,omitempty"`
+
+	// 区域ID
+	CloudregionIds []string `json:"cloudregion_ids"`
+}
+
+type SkuTaskQueryInput struct {
+	// 异步任务ID
+	TaskIds []string `json:"task_ids"`
+}
+
+type CloudregionSkuSyncInput struct {
+	SkuSyncInput
+
+	// 同步资源类型
+	// choices: serversku|elasticcachesku|dbinstance_sku
+	Resource string `json:"resource"`
+}

pkg/apis/compute/storage_const.go

@@ -59,9 +59,10 @@ const (
 	STORAGE_CLOUD_HSSD    = "cloud_hssd"    //增强型SSD云硬盘
 
 	// huawei storage type
-	STORAGE_HUAWEI_SSD  = "SSD"  // 超高IO云硬盘
-	STORAGE_HUAWEI_SAS  = "SAS"  // 高IO云硬盘
-	STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘
+	STORAGE_HUAWEI_SSD   = "SSD"   // 超高IO云硬盘
+	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
+	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
+	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/compute/vpcs_ovn.go

@@ -18,10 +18,13 @@ import (
 	"yunion.io/x/pkg/util/netutils"
 )
 
+const VPC_OVN_ENCAP_COST = 58
+
 const (
 	VPC_EXTERNAL_ACCESS_MODE_DISTGW     = "distgw"     // distgw only
 	VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW = "eip-distgw" // eip when available, distgw otherwise
 	VPC_EXTERNAL_ACCESS_MODE_EIP        = "eip"        // eip only
+	VPC_EXTERNAL_ACCESS_MODE_NONE       = "none"       // no external access
 )
 
 var (
@@ -29,6 +32,7 @@ var (
 		VPC_EXTERNAL_ACCESS_MODE_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP,
+		VPC_EXTERNAL_ACCESS_MODE_NONE,
 	}
 )
 

pkg/apis/identity/consts.go

@@ -99,6 +99,7 @@ var (
 
 	CommonWhitelistOptionMap = map[string][]string{
 		"default": []string{
+			"enable_quota_check",
 			"default_quota_value",
 			"enable_rbac",
 			"non_default_domain_projects",

pkg/apis/input.go

@@ -201,6 +201,8 @@ type PerformStatusInput struct {
 	// 更改的目标状态值
 	// required:true
 	Status string `json:"status"`
+	// swagger:ignore
+	BlockJobsCount int `json:"block_jobs_count"`
 
 	// 更改状态的原因描述
 	// required:false

pkg/apis/monitor/alert.go

@@ -117,6 +117,7 @@ type AlertCreateInput struct {
 	NoDataState string `json:"no_data_state"`
 	// 报警执行错误将当前报警状态设置为对应的状态
 	ExecutionErrorState string `json:"execution_error_state"`
+	UsedBy              string `json:"used_by"`
 }
 
 type AlertUpdateInput struct {

pkg/apis/monitor/alertrecord.go

@@ -34,7 +34,9 @@ type AlertRecordCreateInput struct {
 
 type AlertRecordRule struct {
 	Metric          string `json:"metric"`
+	Measurement     string `json:"measurement"`
 	MeasurementDesc string `json:"measurement_desc"`
+	Field           string `json:"field"`
 	FieldDesc       string `json:"field_desc"`
 	// 比较运算符, 比如: >, <, >=, <=
 	Comparator string `json:"comparator"`

pkg/apis/monitor/suggestsysrule.go

@@ -23,6 +23,11 @@ import (
 const (
 	METRIC_TAG   = "TAG"
 	METRIC_FIELD = "FIELD"
+
+	METRIC_VM_ID    = "vm_id"
+	METRIC_OSS_ID   = "oss_id"
+	METRIC_RDS_ID   = "rds_id"
+	METRIC_REDIS_ID = "redis_id"
 )
 
 var PROPERTY_TYPE = []string{"databases", "measurements", "metric-measurement"}

pkg/apis/scheduler/api.go

@@ -81,6 +81,10 @@ type ScheduleInput struct {
 	CpuMicrocode string `json:"cpu_microcode"`
 	CpuMode      string `json:"cpu_mode"`
 
+	// In the migrate and create backup cases
+	// we don't need reallocate network
+	ReuseNetwork bool `json:"reuse_network"`
+
 	PendingUsages []jsonutils.JSONObject
 }
 

pkg/apis/webconsole/types.go

@@ -0,0 +1,53 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package webconsole
+
+import (
+	"encoding/base64"
+	"net/url"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type ServerRemoteConsoleResponse struct {
+	ConnectParams string `json:"connect_params"`
+	Session       string `json:"session,omitempty"`
+
+	apis.Meta
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectParams() string {
+	params := resp.ConnectParams
+	if data, err := base64.StdEncoding.DecodeString(params); err == nil {
+		params = string(data)
+	}
+	return params
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectProtocol() (string, error) {
+	var (
+		params = resp.GetConnectParams()
+		query  url.Values
+	)
+	if uri, err := url.ParseRequestURI(params); err == nil {
+		query = uri.Query()
+	} else if q, err := url.ParseQuery(params); err == nil {
+		query = q
+	} else {
+		return "", err
+	}
+	protocol := query.Get("protocol")
+	return protocol, nil
+}

pkg/baremetal/cronjobs.go

@@ -93,7 +93,7 @@ func (job *SStatusProbeJob) Name() string {
 
 func (job *SStatusProbeJob) Do(ctx context.Context, now time.Time) error {
 	bStatus := job.baremetal.GetStatus()
-	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING {
+	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING || bStatus == api.BAREMETAL_UNKNOWN {
 		ps, err := job.baremetal.GetPowerStatus()
 		if err != nil {
 			return errors.Wrap(err, "GetPowerStatus")

pkg/baremetal/pxe/dhcp.go

@@ -368,7 +368,7 @@ func (req *dhcpRequest) createBaremetal(session *mcclient.ClientSession) (jsonut
 	params := jsonutils.NewDict()
 	mac := req.ClientMac.String()
 	zoneId := req.baremetalManager.GetZoneId()
-	name := fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1))
+	name := strings.ToLower(fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1)))
 	params.Add(jsonutils.NewString(name), "name")
 	params.Add(jsonutils.NewString(mac), "access_mac")
 	params.Add(jsonutils.NewString("baremetal"), "host_type")

pkg/baremetal/utils/ipmitool/ipmitool.go

@@ -15,6 +15,7 @@
 package ipmitool
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -115,21 +116,21 @@ func (ipmi *LanPlusIPMI) GetMode() string {
 	return "rmcp"
 }
 
-func (ipmi *LanPlusIPMI) GetCommand(args ...string) *procutils.Command {
+func (ipmi *LanPlusIPMI) GetCommand(args ...string) (*procutils.Command, context.CancelFunc) {
 	nArgs := []string{
-		"--signal=KILL",
-		fmt.Sprintf("%s", ipmi.GetDefaultTimeout()),
-		"ipmitool", "-I", "lanplus", "-H", ipmi.host,
+		"-I", "lanplus", "-H", ipmi.host,
 		"-p", fmt.Sprintf("%d", ipmi.port),
 		"-U", ipmi.user,
 		"-P", ipmi.password,
 	}
 	nArgs = append(nArgs, args...)
-	return procutils.NewCommand("timeout", nArgs...)
+	ctx, cancel := context.WithTimeout(context.Background(), ipmi.GetDefaultTimeout())
+	return procutils.NewCommandContext(ctx, "ipmitool", nArgs...), cancel
 }
 
 func (ipmi *LanPlusIPMI) ExecuteCommand(args ...string) ([]string, error) {
-	cmd := ipmi.GetCommand(args...)
+	cmd, cancel := ipmi.GetCommand(args...)
+	defer cancel()
 	log.Debugf("[LanPlusIPMI] execute command: %s", cmd.String())
 	out, err := cmd.Output()
 	if err != nil {

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -624,7 +624,7 @@ func (a StorcliAdaptor) key() string {
 }
 
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.key() != ""
+	return a.Controller >= 0 && a.name != "" && a.sn != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {
@@ -756,7 +756,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build JBOD fail: %v", err)
+	log.Errorf("Try storcli build JBOD fail: %v", err)
 	labels := []string{}
 	for _, dev := range devs {
 		labels = append(labels, GetSpecString(dev))
@@ -779,7 +779,7 @@ func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build jbod fail: %v", err)
+	log.Errorf("Try megacli build jbod fail: %v", err)
 	cmds := []string{}
 	for _, dev := range devs {
 		cmd := GetCommand("-CfgLdAdd", "-r0", fmt.Sprintf("[%s]", GetSpecString(dev)),
@@ -862,29 +862,42 @@ func (adapter *MegaRaidAdaptor) RemoveLogicVolumes() error {
 	return nil
 }
 
-/*
-def _storcli_clear_jbod_disks(self):
-    cmds = []
-    for dev in self.devs:
-        cmd = self.raid.get_command2(
-                    '/c%d/e%d/s%d' % (self.index, dev.enclosure, dev.slot),
-                    'set', 'good', 'force')
-        cmds.append(cmd)
-    logging.info('%s', cmds)
-    self.raid.term.exec_remote_commands(cmds)
-*/
+func (adapter *MegaRaidAdaptor) storcliClearJBODDisks() error {
+	errs := make([]error, 0)
+	for _, dev := range adapter.devs {
+		cmd := GetCommand2(fmt.Sprintf("/c%d/e%d/s%d", adapter.index, dev.enclosure, dev.slot), "set", "good", "force")
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "Set PD good storcli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
+}
 
 func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
 	devIds := []string{}
 	for idx, dev := range adapter.devs {
 		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage(idx)))
 	}
-	cmd := GetCommand("-PDMakeGood", fmt.Sprintf("-PhysDrv[%s]", strings.Join(devIds, ",")), "-Force", fmt.Sprintf("-a%d", adapter.index))
-	_, err := adapter.remoteRun(cmd)
-	return err
+	errs := make([]error, 0)
+	for _, devId := range devIds {
+		cmd := GetCommand("-PDMakeGood", "-PhysDrv", fmt.Sprintf("'[%s]'", devId), "-Force", fmt.Sprintf("-a%d", adapter.index))
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "PDMakeGood megacli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
 }
 
 func (adapter *MegaRaidAdaptor) clearJBODDisks() {
+	if err := adapter.megacliClearJBODDisks(); err != nil {
+		log.Errorf("megacliClearJBODDisks error: %v", err)
+		log.Infof("try storcliClearJBODDisks")
+		if err := adapter.storcliClearJBODDisks(); err != nil {
+			log.Errorf("storcliClearJBODDisks error: %v", err)
+		}
+	}
 	adapter.megacliEnableJBOD(true)
 	adapter.megacliEnableJBOD(false)
 	adapter.megacliEnableJBOD(true)

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -56,7 +56,7 @@ func (dev *Mpt2SASRaidPhyDev) parseLine(line string) bool {
 	}
 	switch key {
 	case "Drive Type":
-		if strings.HasSuffix("_HDD", val) {
+		if strings.HasSuffix(val, "_HDD") {
 			dev.Rotate = tristate.True
 		} else {
 			dev.Rotate = tristate.False

pkg/cloudcommon/consts/consts.go

@@ -35,6 +35,8 @@ var (
 	maxPagingLimit     int64 = 2048
 
 	domainizedNamespace = true
+
+	enableQuotaCheck = false
 )
 
 func SetRegion(region string) {
@@ -85,3 +87,11 @@ func SetDomainizedNamespace(domainNS bool) {
 func IsDomainizedNamespace() bool {
 	return nonDefaultDomainProjects && domainizedNamespace
 }
+
+func SetEnableQuotaCheck(val bool) {
+	enableQuotaCheck = val
+}
+
+func EnableQuotaCheck() bool {
+	return enableQuotaCheck
+}

pkg/cloudcommon/db/jointbase.go

@@ -34,7 +34,7 @@ import (
 type SJointResourceBase struct {
 	SResourceBase
 
-	RowId int64 `primary:"true" auto_increment:"true"`
+	RowId int64 `primary:"true" auto_increment:"true" list:"user"`
 }
 
 type SJointResourceBaseManager struct {

pkg/cloudcommon/db/metadata.go

@@ -459,17 +459,17 @@ func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key s
 }
 
 func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
 		return err
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
 
-func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
+func (manager *SMetadataManager) setValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
 	// no need to lock
@@ -546,9 +546,9 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 }
 
 func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "setValues")
 	}
 
 	idStr := GetObjectIdstr(obj)
@@ -565,9 +565,9 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 	q := manager.Query().Equals("id", idStr).NotLike("key", `\_\_%`) //避免删除系统内置的metadata, _ 在mysql里面有特殊含义,需要转义
 	switch delRange {
 	case USER_TAG_PREFIX:
-		q = q.Like("key", USER_TAG_PREFIX+"%")
+		q = q.Startswith("key", USER_TAG_PREFIX)
 	case CLOUD_TAG_PREFIX:
-		q = q.Like("key", CLOUD_TAG_PREFIX+"%")
+		q = q.Startswith("key", CLOUD_TAG_PREFIX)
 	}
 	q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys)))
 	if err := FetchModelObjects(manager, q, &records); err != nil {
@@ -581,7 +581,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }

pkg/cloudcommon/db/opslog.go

@@ -117,7 +117,8 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
 	if !consts.OpsLogEnabled() {
 		return
 	}
-	if len(model.GetId()) == 0 || len(model.GetName()) == 0 {
+	if len(model.GetId()) == 0 {
+		log.Errorf("logevent for an object without ID???")
 		return
 	}
 	if action == ACT_UPDATE {

pkg/cloudcommon/db/opslog_const.go

@@ -47,8 +47,9 @@ const (
 	ACT_BACKUP_START        = "backup_start"
 	ACT_BACKUP_START_FAILED = "backup_start_fail"
 
-	ACT_FREEZE   = "freeze"
-	ACT_UNFREEZE = "unfreeze"
+	ACT_FREEZE      = "freeze"
+	ACT_FREEZE_FAIL = "freeze_fail"
+	ACT_UNFREEZE    = "unfreeze"
 
 	ACT_RESTARING    = "restarting"
 	ACT_RESTART_FAIL = "restart_fail"
@@ -260,4 +261,6 @@ const (
 
 	ACT_SYNC_VPCS        = "sync_vpcs"
 	ACT_SYNC_RECORD_SETS = "sync_record_sets"
+
+	ACT_UPDATE_RULE = "update_config"
 )

pkg/cloudcommon/db/quotas/errors.go

@@ -19,9 +19,11 @@ import (
 	"strings"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SOutOfQuotaError struct {
+	scope   rbacutils.TRbacScope
 	name    string
 	limit   int
 	used    int
@@ -37,7 +39,7 @@ func (e *SOutOfQuotaError) Cause() error {
 }
 
 func (e *SOutOfQuotaError) Error() string {
-	return fmt.Sprintf("%s limit %d used %d request %d", e.name, e.limit, e.used, e.request)
+	return fmt.Sprintf("[%s.%s] limit %d used %d request %d", e.scope, e.name, e.limit, e.used, e.request)
 }
 
 func (es *SOutOfQuotaErrors) Error() string {
@@ -63,8 +65,10 @@ func NewOutOfQuotaError() *SOutOfQuotaErrors {
 	}
 }
 
-func (es *SOutOfQuotaErrors) Add(name string, limit int, used int, request int) {
+func (es *SOutOfQuotaErrors) Add(quota IQuota, name string, limit int, used int, request int) {
+	scope := quota.GetKeys().Scope()
 	e := SOutOfQuotaError{
+		scope:   scope,
 		name:    name,
 		limit:   limit,
 		used:    used,

pkg/cloudcommon/db/quotas/handler.go

@@ -132,6 +132,12 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 	ret := jsonutils.NewDict()
 
 	keys := quota.GetKeys()
+	ret.Update(jsonutils.Marshal(keys))
+	ret.Update(quota.ToJSON(""))
+
+	if !consts.EnableQuotaCheck() {
+		return ret, nil
+	}
 
 	usage := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, usage)
@@ -150,8 +156,6 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 		return nil, errors.Wrap(err, "manager.GetPendingUsages")
 	}
 
-	ret.Update(jsonutils.Marshal(keys))
-	ret.Update(quota.ToJSON(""))
 	if usage != nil {
 		ret.Update(usage.ToJSON("usage"))
 	}

pkg/cloudcommon/db/quotas/interface.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type IQuotaKeys interface {
@@ -30,6 +31,8 @@ type IQuotaKeys interface {
 	Compare(IQuotaKeys) int
 
 	OwnerId() mcclient.IIdentityProvider
+
+	Scope() rbacutils.TRbacScope
 }
 
 type IQuota interface {

pkg/cloudcommon/db/quotas/register.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -57,6 +58,10 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 }
 
 func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	if localUsage == nil {
 		return nil
 	}
@@ -65,6 +70,10 @@ func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential,
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	manager := getQuotaManager(quota)
 	err := manager.checkSetPendingQuota(ctx, userCred, quota)
 	if err != nil {
@@ -75,6 +84,9 @@ func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		cancelUsage(ctx, userCred, usage.(IQuota))
 	}
@@ -89,6 +101,9 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 }
 
 func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		addUsage(ctx, userCred, usage.(IQuota))
 	}

pkg/cloudcommon/db/quotas/usageworker.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -61,6 +62,12 @@ func isDirty(key string) bool {
 }
 
 func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan IQuota, realTime bool) {
+	if !consts.EnableQuotaCheck() {
+		go func() {
+			usageChan <- nil
+		}()
+		return
+	}
 	key := QuotaKeyString(keys)
 	setDirty(key)
 
@@ -105,6 +112,10 @@ func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan I
 }
 
 func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
+
 	log.Infof("CalculateQuotaUsages")
 	quota := manager.newQuota()
 	keys := quota.GetKeys()

pkg/cloudcommon/db/rbac.go

@@ -51,6 +51,8 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeUser:
 		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() || objOwnerId.GetUserId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && objOwnerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}
@@ -103,6 +105,8 @@ func isClassRbacAllowed(manager IModelManager, userCred mcclient.TokenCredential
 	case rbacutils.ScopeUser:
 		if ownerId != nil && ownerId.GetUserId() == objOwnerId.GetUserId() {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}

pkg/cloudcommon/db/sharablebase.go

@@ -472,6 +472,10 @@ func SharablePerformPublic(model ISharableBaseModel, ctx context.Context, userCr
 	if targetScope.HigherThan(allowScope) {
 		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", targetScope, allowScope)
 	}
+	requireScope := model.GetPublicScope()
+	if requireScope.HigherThan(allowScope) {
+		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", requireScope, allowScope)
+	}
 
 	_, err = Update(model, func() error {
 		model.SetShare(targetScope)

pkg/cloudcommon/db/taskman/localtaskworker.go

@@ -37,8 +37,8 @@ func Error2TaskData(err error) jsonutils.JSONObject {
 	return errJson
 }
 
-func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
-	localTaskWorkerMan.Run(func() {
+func LocalTaskRunWithWorkers(task ITask, proc func() (jsonutils.JSONObject, error), wm *appsrv.SWorkerManager) {
+	wm.Run(func() {
 
 		log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN STARTXXXXXXXXXXXXXXXXX")
 		defer log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN END  XXXXXXXXXXXXXXXXX")
@@ -59,3 +59,7 @@ func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
 
 	}, nil, nil)
 }
+
+func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
+	LocalTaskRunWithWorkers(task, proc, localTaskWorkerMan)
+}

pkg/cloudcommon/db/taskman/tasks.go

@@ -41,6 +41,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
@@ -280,8 +281,9 @@ func (manager *STaskManager) NewTask(
 	}
 	parentTask := task.GetParentTask()
 	if parentTask != nil {
-		st := SSubTask{TaskId: parentTask.Id, Stage: parentTask.Stage, SubtaskId: task.Id}
-		err := SubTaskManager.TableSpec().Insert(ctx, &st)
+		st := &SSubTask{TaskId: parentTask.Id, Stage: parentTask.Stage, SubtaskId: task.Id}
+		st.SetModelManager(SubTaskManager, st)
+		err := SubTaskManager.TableSpec().Insert(ctx, st)
 		if err != nil {
 			log.Errorf("Subtask insert error %s", err)
 			return nil, err
@@ -814,7 +816,11 @@ func (self *STask) GetObjects() []db.IStandaloneModel {
 }
 
 func (task *STask) GetTaskRequestHeader() http.Header {
-	header := mcclient.GetTokenHeaders(task.GetUserCred())
+	userCred := task.GetUserCred()
+	if !userCred.IsValid() {
+		userCred = auth.AdminCredential()
+	}
+	header := mcclient.GetTokenHeaders(userCred)
 	header.Set(mcclient.TASK_ID, task.GetTaskId())
 	return header
 }

pkg/cloudcommon/etcd/etcd.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
@@ -45,7 +46,8 @@ type SEtcdClient struct {
 	onKeepaliveFailure func()
 	leaseLiving        bool
 
-	watchers map[string]*SEtcdWatcher
+	watchers   map[string]*SEtcdWatcher
+	watchersMu *sync.Mutex
 }
 
 func defaultOnKeepAliveFailed() {
@@ -115,6 +117,7 @@ func NewEtcdClient(opt *SEtcdOptions, onKeepaliveFailure func()) (*SEtcdClient,
 	etcdClient.leaseTtlTimeout = timeoutSeconds
 
 	etcdClient.watchers = make(map[string]*SEtcdWatcher)
+	etcdClient.watchersMu = &sync.Mutex{}
 
 	etcdClient.namespace = opt.EtcdNamspace
 
@@ -143,6 +146,10 @@ func (cli *SEtcdClient) GetClient() *clientv3.Client {
 	return cli.client
 }
 
+func (cli *SEtcdClient) SessionLiving() bool {
+	return cli.leaseLiving
+}
+
 func (cli *SEtcdClient) startSession() error {
 	ctx := context.TODO()
 
@@ -302,8 +309,10 @@ func (cli *SEtcdClient) Watch(
 	onModify TEtcdModifyEventFunc,
 	onDelete TEtcdDeleteEventFunc,
 ) error {
+	cli.watchersMu.Lock()
 	_, ok := cli.watchers[prefix]
 	if ok {
+		cli.watchersMu.Unlock()
 		return errors.Errorf("watch prefix %s already registered", prefix)
 	}
 
@@ -314,6 +323,7 @@ func (cli *SEtcdClient) Watch(
 		watcher: watcher,
 		cancel:  cancel,
 	}
+	cli.watchersMu.Unlock()
 
 	prefix = cli.getKey(prefix)
 
@@ -342,6 +352,8 @@ func (cli *SEtcdClient) Watch(
 }
 
 func (cli *SEtcdClient) Unwatch(prefix string) {
+	cli.watchersMu.Lock()
+	defer cli.watchersMu.Unlock()
 	watcher, ok := cli.watchers[prefix]
 	if ok {
 		log.Debugf("unwatch %s", prefix)

pkg/cloudcommon/notifyclient/notify.go

@@ -91,7 +91,9 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 		if err != nil {
 			return nil, err
 		}
-		tmp, err := template.New(key).Parse(string(cont))
+		tmp := template.New(key)
+		tmp.Funcs(template.FuncMap{"unescaped": unescaped})
+		tmp, err = tmp.Parse(string(cont))
 		if err != nil {
 			return nil, err
 		}
@@ -100,6 +102,10 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 	return templatesTable[key], nil
 }
 
+func unescaped(str string) template.HTML {
+	return template.HTML(str)
+}
+
 func getContent(ctx context.Context, topic string, contType string, channel npk.TNotifyChannel, data jsonutils.JSONObject) (string, error) {
 	if channel == npk.NotifyByWebhook {
 		return "", nil

pkg/cloudcommon/options/changes.go

@@ -40,6 +40,9 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 		consts.SetDomainizedNamespace(newOpts.DomainizedNamespace)
 		changed = true
 	}
+	if oldOpts.EnableQuotaCheck != newOpts.EnableQuotaCheck {
+		consts.SetEnableQuotaCheck(newOpts.EnableQuotaCheck)
+	}
 	return changed
 }
 

pkg/cloudcommon/options/manager.go

@@ -124,10 +124,10 @@ func (manager *SOptionManager) DoSync(first bool) (time.Duration, error) {
 			appsrv.SetExitFlag()
 		}
 		copyOptions(manager.options, newOpts)
-		if first {
-			// upload config for the first time ONLY
-			manager.session.Upload()
-		}
+		// if first {
+		// upload config
+		manager.session.Upload()
+		// }
 	}
 	return manager.refreshInterval, nil
 }

pkg/cloudcommon/options/mergeconf.go

@@ -66,6 +66,8 @@ type mcclientServiceConfigSession struct {
 	session   *mcclient.ClientSession
 	serviceId string
 	config    *jsonutils.JSONDict
+
+	commonServiceId string
 }
 
 func newServiceConfigSession() IServiceConfigSession {
@@ -88,12 +90,12 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 			merged = true
 		} else {
 			// not initialized
-			s.Upload()
+			// s.Upload()
 		}
 	}
-	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
-	if len(commonServiceId) > 0 {
-		commonConf, err := getServiceConfig(s.session, commonServiceId)
+	s.commonServiceId, _ = getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
+	if len(s.commonServiceId) > 0 {
+		commonConf, err := getServiceConfig(s.session, s.commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
 		} else if commonConf != nil {
@@ -123,6 +125,11 @@ func (s *mcclientServiceConfigSession) Upload() {
 			// ignore the error
 			log.Errorf("fail to save config: %s", err)
 		}
+		_, err = modules.ServicesV3.PerformAction(s.session, s.commonServiceId, "config", nconf)
+		if err != nil {
+			// ignore the error
+			log.Errorf("fail to save common config: %s", err)
+		}
 	}
 }
 

pkg/cloudcommon/options/options.go

@@ -78,6 +78,7 @@ type BaseOptions struct {
 	IsSlaveNode        bool `help:"Slave mode"`
 	CronJobWorkerCount int  `help:"Cron job worker count" default:"4"`
 
+	EnableQuotaCheck  bool   `help:"enable quota check" default:"true"`
 	DefaultQuotaValue string `help:"default quota value" choices:"unlimit|zero|default" default:"default"`
 
 	CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"`

pkg/cloudcommon/validators/validators.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/regutils"
@@ -821,3 +822,15 @@ func NewIPv4AddrValidator(key string) *ValidatorIPv4Addr {
 	v.SetParent(v)
 	return v
 }
+
+var ValidateModel = func(userCred mcclient.TokenCredential, manager db.IStandaloneModelManager, id *string) (db.IModel, error) {
+	model, err := manager.FetchByIdOrName(userCred, *id)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), *id)
+		}
+		return nil, httperrors.NewGeneralError(err)
+	}
+	*id = model.GetId()
+	return model, nil
+}

pkg/cloudevent/models/cloudproviders.go

@@ -262,23 +262,12 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	if err != nil {
 		return start, end, errors.Wrap(err, "self.GetProviderFactory")
 	}
-	q := CloudeventManager.Query().Equals("cloudprovider_id", self.Id).Desc("created_at")
-	count, err := q.CountWithError()
-	if err != nil {
-		return start, end, errors.Wrap(err, "q.CountWithError")
-	}
 	if !self.LastSyncTimeAt.IsZero() {
 		start = self.LastSyncTimeAt
-	} else if count == 0 {
-		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	} else {
-		event := &SCloudevent{}
-		err = q.First(event)
-		if err != nil {
-			return start, end, errors.Wrap(err, "q.First")
-		}
-		start = event.CreatedAt
+		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	}
+
 	// 避免cloudevent过长时间未运行，再次运行时记录的最后一条时间距离现在间隔太长
 	if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
 		start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)

pkg/cloudevent/service/handlers.go

@@ -55,6 +55,8 @@ func InitHandlers(app *appsrv.Application) {
 
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
+		db.Metadata,
+
 		proxy.ProxySettingManager,
 		models.CloudeventManager,
 	} {

pkg/cloudid/models/cloudaccount.go

@@ -168,6 +168,7 @@ func (manager *SCloudaccountManager) syncCloudaccounts(ctx context.Context, user
 			continue
 		}
 		account.StartSyncCloudIdResourcesTask(ctx, userCred, "")
+		account.StartSystemCloudpolicySyncTask(ctx, userCred, false, "") // 避免新加账号未能及时同步策略
 		localAccounts = append(localAccounts, *account)
 		result.Add()
 	}
@@ -676,7 +677,16 @@ func (self *SCloudaccount) GetCloudaccountByProvider(provider string) ([]SClouda
 	return accounts, nil
 }
 
-func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential) error {
+func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error {
+	dbPolicies, err := self.GetSystemCloudpolicies()
+	if err != nil {
+		return errors.Wrapf(err, "GetSystemCloudpolicies")
+	}
+
+	if len(dbPolicies) > 0 && !refresh {
+		return nil
+	}
+
 	factory, err := self.GetProviderFactory()
 	if err != nil {
 		return errors.Wrapf(err, "GetProviderFactory")
@@ -716,22 +726,18 @@ func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context,
 			return errors.Wrapf(err, "GetISystemCloudpolicies for account %s(%s)", self.Name, self.Provider)
 		}
 	}
-	return self.syncSystemCloudpoliciesFromCloud(ctx, userCred, iPolicies)
+	return self.syncSystemCloudpoliciesFromCloud(ctx, userCred, iPolicies, dbPolicies)
 }
 
-func (self *SCloudaccount) syncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, iPolicies []cloudprovider.ICloudpolicy) error {
+func (self *SCloudaccount) syncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, iPolicies []cloudprovider.ICloudpolicy, dbPolicies []SCloudpolicy) error {
 	result := compare.SyncResult{}
-	dbPolicies, err := self.GetSystemCloudpolicies()
-	if err != nil {
-		return errors.Wrapf(err, "GetSystemCloudpolicies")
-	}
 
 	removed := make([]SCloudpolicy, 0)
 	commondb := make([]SCloudpolicy, 0)
 	commonext := make([]cloudprovider.ICloudpolicy, 0)
 	added := make([]cloudprovider.ICloudpolicy, 0)
 
-	err = compare.CompareSets(dbPolicies, iPolicies, &removed, &commondb, &commonext, &added)
+	err := compare.CompareSets(dbPolicies, iPolicies, &removed, &commondb, &commonext, &added)
 	if err != nil {
 		return errors.Wrapf(err, "compare.CompareSets")
 	}
@@ -832,6 +838,9 @@ func (self *SCloudaccount) GetICloudprovider() ([]SCloudprovider, error) {
 }
 
 func (self *SCloudaccount) syncCloudprovider(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
+	lockman.LockRawObject(ctx, "cloudproviders", self.Id)
+	defer lockman.ReleaseRawObject(ctx, "cloudproviders", self.Id)
+
 	result := compare.SyncResult{}
 
 	providers, err := self.GetICloudprovider()
@@ -932,22 +941,28 @@ func (manager *SCloudaccountManager) SyncCloudidSystemPolicies(ctx context.Conte
 		log.Errorf("GetSupportCloudIdAccounts error: %v", err)
 		return
 	}
+	providers := []string{}
 	for i := range accounts {
 		_, err := accounts[i].GetProvider() // 检查账号是否可以正常连接
 		if err != nil {
 			log.Errorf("GetProvider for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err)
 			continue
 		}
-		err = accounts[i].StartSystemCloudpolicySyncTask(ctx, userCred, "")
+		if utils.IsInStringArray(accounts[i].Provider, providers) {
+			continue
+		}
+		err = accounts[i].StartSystemCloudpolicySyncTask(ctx, userCred, true, "")
 		if err != nil {
 			log.Errorf("StartSystemCloudpolicySyncTask for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err)
 			continue
 		}
+		providers = append(providers, accounts[i].Provider)
 	}
 }
 
-func (self *SCloudaccount) StartSystemCloudpolicySyncTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+func (self *SCloudaccount) StartSystemCloudpolicySyncTask(ctx context.Context, userCred mcclient.TokenCredential, refresh bool, parentTaskId string) error {
 	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewBool(refresh), "refresh")
 	task, err := taskman.TaskManager.NewTask(ctx, "SystemCloudpolicySyncTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
 		return errors.Wrap(err, "NewTask")

pkg/cloudid/tasks/sync_cloudid_system_policies_task.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudid/models"
 )
@@ -39,7 +40,12 @@ func (self *SystemCloudpolicySyncTask) taskFailed(ctx context.Context, cloudacco
 
 func (self *SystemCloudpolicySyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
 	account := obj.(*models.SCloudaccount)
-	err := account.SyncSystemCloudpoliciesFromCloud(ctx, self.GetUserCred())
+
+	lockman.LockRawObject(ctx, "system policies", account.Provider)
+	defer lockman.ReleaseRawObject(ctx, "system policies", account.Provider)
+
+	refresh := jsonutils.QueryBoolean(self.GetParams(), "refresh", false)
+	err := account.SyncSystemCloudpoliciesFromCloud(ctx, self.GetUserCred(), refresh)
 	if err != nil {
 		self.taskFailed(ctx, account, errors.Wrapf(err, "SyncSystemCloudpoliciesFromCloud"))
 		return

pkg/cloudprovider/dbinstance.go

@@ -16,6 +16,14 @@ package cloudprovider
 
 import "yunion.io/x/onecloud/pkg/util/billing"
 
+type TBackupMethod string
+
+const (
+	BackupMethodLogical  = TBackupMethod("Logical")
+	BackupMethodPhysical = TBackupMethod("Physical")
+	BackupMethodUnknown  = TBackupMethod("")
+)
+
 type SDBInstanceNetwork struct {
 	IP        string
 	NetworkId string
@@ -55,6 +63,9 @@ type SManagedDBInstanceCreateConfig struct {
 	ProjectId        string
 
 	BillingCycle *billing.SBillingCycle
+
+	// 仅从备份恢复到新实例用到
+	RdsId string
 }
 
 type SManagedDBInstanceChangeConfig struct {

pkg/cloudprovider/instance.go

@@ -26,6 +26,52 @@ import (
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )
 
+type SDistDefaultAccount struct {
+	// 操作系统发行版
+	OsDistribution string
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改
+	Changeable bool
+}
+
+type SOsDefaultAccount struct {
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改用户名
+	Changeable bool
+	// 禁止使用的账号
+	DisabledAccounts []string
+	// 各操作系统发行版的默认用户名信息
+	DistAccounts []SDistDefaultAccount
+}
+
+type SDefaultAccount struct {
+	Linux   SOsDefaultAccount
+	Windows SOsDefaultAccount
+}
+
+type StorageInfo struct {
+	StorageType string
+	MaxSizeGb   int
+	MinSizeGb   int
+	Resizable   bool
+	StepSizeGb  int
+}
+
+type Storage struct {
+	DataDisk []StorageInfo
+	SysDisk  []StorageInfo
+}
+
+type SInstanceCapability struct {
+	Provider   string
+	Hypervisor string
+	Storages   Storage
+
+	DefaultAccount SDefaultAccount
+}
+
 type SDiskInfo struct {
 	StorageExternalId string
 	StorageType       string

pkg/cloudprovider/resources.go

@@ -75,6 +75,7 @@ type ICloudRegion interface {
 	CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error)
 
 	CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error)
+	CreateInternetGateway() (ICloudInternetGateway, error)
 	CreateEIP(eip *SEip) (ICloudEIP, error)
 
 	GetISnapshots() ([]ICloudSnapshot, error)
@@ -437,6 +438,10 @@ type ICloudVpc interface {
 	// GetGlobalId() // 若vpc属于globalvpc,此函数返回格式必须是 'region.GetGlobalId()/vpc.GetGlobalId()'
 	ICloudResource
 
+	IsSupportSetExternalAccess() bool // 是否支持Attach互联网网关.
+	GetExternalAccessMode() string
+	AttachInternetGateway(igwId string) error
+
 	GetRegion() ICloudRegion
 	GetIsDefault() bool
 	GetCidrBlock() string
@@ -451,6 +456,10 @@ type ICloudVpc interface {
 	GetINatGateways() ([]ICloudNatGateway, error)
 }
 
+type ICloudInternetGateway interface {
+	ICloudResource
+}
+
 type ICloudWire interface {
 	ICloudResource
 	GetIVpc() ICloudVpc
@@ -816,6 +825,9 @@ type ICloudDBInstanceBackup interface {
 	GetBackupSizeMb() int
 	GetDBNames() string
 	GetBackupMode() string
+	GetBackupMethod() TBackupMethod
+
+	CreateICloudDBInstance(opts *SManagedDBInstanceCreateConfig) (ICloudDBInstance, error)
 
 	Delete() error
 }

pkg/cloudprovider/securitygroup.go

@@ -18,12 +18,64 @@ import (
 	"sort"
 	"strings"
 
+	"gopkg.in/fatih/set.v0"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/secrules"
+	"yunion.io/x/pkg/utils"
 )
 
-const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
-const DEFAULT_LOCAL_RULE_ID = "default_local_rule_id"
+type SecDriver interface {
+	GetDefaultSecurityGroupInRule() SecurityRule
+	GetDefaultSecurityGroupOutRule() SecurityRule
+	GetSecurityGroupRuleMaxPriority() int
+	GetSecurityGroupRuleMinPriority() int
+	IsOnlySupportAllowRules() bool
+}
+
+func NewSecRuleInfo(driver SecDriver) SecRuleInfo {
+	return SecRuleInfo{
+		InDefaultRule:           driver.GetDefaultSecurityGroupInRule(),
+		OutDefaultRule:          driver.GetDefaultSecurityGroupOutRule(),
+		MinPriority:             driver.GetSecurityGroupRuleMinPriority(),
+		MaxPriority:             driver.GetSecurityGroupRuleMaxPriority(),
+		IsOnlySupportAllowRules: driver.IsOnlySupportAllowRules(),
+	}
+}
+
+const DEFAULT_DEST_RULE_ID = "default_dest_rule_id"
+const DEFAULT_SRC_RULE_ID = "default_src_rule_id"
+
+type SecRuleInfo struct {
+	InDefaultRule           SecurityRule
+	OutDefaultRule          SecurityRule
+	Rules                   SecurityRuleSet
+	MinPriority             int
+	MaxPriority             int
+	IsOnlySupportAllowRules bool
+}
+
+func (r SecRuleInfo) AddDefaultRule(d SecRuleInfo, inRules, outRules []SecurityRule, isSrc bool) ([]SecurityRule, []SecurityRule) {
+	min, max := r.MinPriority, r.MaxPriority
+	r.InDefaultRule.Priority = min + 1
+	r.OutDefaultRule.Priority = min + 1
+	if max >= min {
+		r.InDefaultRule.Priority = min - 1
+		r.OutDefaultRule.Priority = min - 1
+	}
+
+	if isSrc {
+		r.InDefaultRule.Id = DEFAULT_SRC_RULE_ID
+		r.OutDefaultRule.Id = DEFAULT_SRC_RULE_ID
+	} else {
+		r.InDefaultRule.ExternalId = DEFAULT_DEST_RULE_ID
+		r.OutDefaultRule.ExternalId = DEFAULT_DEST_RULE_ID
+	}
+
+	inRules = append(inRules, r.InDefaultRule)
+	outRules = append(outRules, r.OutDefaultRule)
+	return inRules, outRules
+}
 
 type SecurityGroupFilterOptions struct {
 	VpcId     string
@@ -43,57 +95,26 @@ type SecurityRule struct {
 	secrules.SecurityRule
 	Name       string
 	ExternalId string
+	Id         string
 }
 
-type LocalSecurityRule struct {
-	secrules.SecurityRule
-	ExternalId string
-}
-
-func (r LocalSecurityRule) String() string {
-	return r.SecurityRule.String()
-}
-
-type LocalSecurityRuleSet []LocalSecurityRule
-
-func (srs LocalSecurityRuleSet) Len() int {
-	return len(srs)
-}
-
-func (srs LocalSecurityRuleSet) Swap(i, j int) {
-	srs[i], srs[j] = srs[j], srs[i]
-}
-
-func (srs LocalSecurityRuleSet) Less(i, j int) bool {
-	if srs[i].Priority > srs[j].Priority {
-		return true
-	} else if srs[i].Priority == srs[j].Priority {
-		return srs[i].String() < srs[j].String()
-	}
-	return false
-}
-
-func (srs LocalSecurityRuleSet) AllowList() secrules.SecurityRuleSet {
-	rules := secrules.SecurityRuleSet{}
-	for _, r := range srs {
-		rules = append(rules, r.SecurityRule)
-	}
-	return rules.AllowList()
-}
-
-type TPriorityOrder int
-
-var (
-	PriorityOrderByDesc = TPriorityOrder(1)
-	PriorityOrderByAsc  = TPriorityOrder(-1)
-)
-
 func (r SecurityRule) String() string {
 	return r.SecurityRule.String()
 }
 
 type SecurityRuleSet []SecurityRule
 
+func (rules SecurityRuleSet) Split() (in, out SecurityRuleSet) {
+	for i := 0; i < len(rules); i++ {
+		if rules[i].Direction == secrules.DIR_IN {
+			in = append(in, rules[i])
+		} else {
+			out = append(out, rules[i])
+		}
+	}
+	return
+}
+
 func (srs SecurityRuleSet) Len() int {
 	return len(srs)
 }
@@ -114,234 +135,191 @@ func (srs SecurityRuleSet) AllowList() secrules.SecurityRuleSet {
 	return rules.AllowList()
 }
 
-func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr string, order TPriorityOrder, min, max int, onlyAllowRules bool) []SecurityRule {
-	if defaultRule.String() == localRuleStr {
-		return rules
+func (srs SecurityRuleSet) Debug() {
+	for i := 0; i < len(srs); i++ {
+		log.Debugf("Name: %s id: %s external_id: %s priority: %d %s", srs[i].Name, srs[i].Id, srs[i].ExternalId, srs[i].Priority, srs[i].String())
 	}
-	defaultRule.ExternalId = DEFAULT_CLOUD_RULE_ID
-	if order == PriorityOrderByDesc {
-		defaultRule.Priority = min
-	} else {
-		defaultRule.Priority = max
-	}
-	defaultRule.Priority -= int(order)
-	return append(rules, defaultRule)
 }
 
-func SortSecurityRule(rules SecurityRuleSet, order TPriorityOrder, onlyAllowRules bool) {
-	if onlyAllowRules {
-		sort.Sort(rules)
-		return
-	}
-	if order == PriorityOrderByAsc {
+func SortSecurityRule(rules SecurityRuleSet, max, min int, isAsc, onlyAllowRules bool) {
+	if (max >= min || onlyAllowRules) && !isAsc {
 		sort.Sort(sort.Reverse(rules))
 		return
 	}
 	sort.Sort(rules)
+	return
 }
 
-func CompareRules(
-	minPriority, maxPriority int, order TPriorityOrder,
-	localRules secrules.SecurityRuleSet, remoteRules []SecurityRule,
-	defaultInRule, defaultOutRule SecurityRule,
-	onlyAllowRules bool, debug bool,
-) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
-	localInRules := LocalSecurityRuleSet{}
-	localOutRules := LocalSecurityRuleSet{}
-	for i := range localRules {
-		localRule := LocalSecurityRule{}
-		localRule.SecurityRule = localRules[i]
-		if localRules[i].Direction == secrules.DIR_IN {
-			localInRules = append(localInRules, localRule)
-		} else {
-			localOutRules = append(localOutRules, localRule)
-		}
+func isAllowListEqual(src, dest secrules.SecurityRuleSet) bool {
+	if len(src) != len(dest) {
+		return false
 	}
-	inRules := SecurityRuleSet{}
-	outRules := SecurityRuleSet{}
-	for i := 0; i < len(remoteRules); i++ {
-		if remoteRules[i].Direction == secrules.DIR_IN {
-			inRules = append(inRules, remoteRules[i])
-		} else {
-			outRules = append(outRules, remoteRules[i])
-		}
+	s1, s2 := set.New(set.ThreadSafe), set.New(set.ThreadSafe)
+	for i := 0; i < len(src); i++ {
+		s1.Add(src[i].String())
+		s2.Add(dest[i].String())
 	}
-	var inCommon, outCommon = inRules, outRules
+	return s1.IsEqual(s2)
+}
 
-	defaultLocalInRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
-	defaultLocalInRule.SecurityRule = *secrules.MustParseSecurityRule("in:deny any")
-	defaultLocalOutRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
-	defaultLocalOutRule.SecurityRule = *secrules.MustParseSecurityRule("out:allow any")
+func CompareRules(src, dest SecRuleInfo, debug bool) (common, inAdds, outAdds, inDels, outDels SecurityRuleSet) {
+	srcInRules, srcOutRules := src.Rules.Split()
+	destInRules, destOutRules := dest.Rules.Split()
 
-	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
-	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+	srcInRules, srcOutRules = src.AddDefaultRule(dest, srcInRules, srcOutRules, true)
+	destInRules, destOutRules = dest.AddDefaultRule(src, destInRules, destOutRules, false)
 
-	defaultInEquals, defaultOutEquals := true, true
-	if defaultLocalInRule.String() != defaultInRule.String() {
-		localInRules = append(localInRules, defaultLocalInRule)
-		defaultInEquals = false
-	}
-	if defaultLocalOutRule.String() != defaultOutRule.String() {
-		localOutRules = append(localOutRules, defaultLocalOutRule)
-		defaultOutEquals = false
+	if debug {
+		log.Debugf("src in rules: ")
+		srcInRules.Debug()
 	}
 
-	sort.Sort(localInRules)
-	sort.Sort(localOutRules)
+	// AllowList 需要优先级从高到低排序
+	SortSecurityRule(srcInRules, src.MaxPriority, src.MinPriority, false, src.IsOnlySupportAllowRules)
+	SortSecurityRule(srcOutRules, src.MaxPriority, src.MinPriority, false, src.IsOnlySupportAllowRules)
 
-	localInAllowList := localInRules.AllowList()
-	localOutAllowList := localOutRules.AllowList()
-	_localInRules := LocalSecurityRuleSet{}
-	for i := range localInAllowList {
-		rule := LocalSecurityRule{}
-		rule.SecurityRule = localInAllowList[i]
-		_localInRules = append(_localInRules, rule)
-	}
-	_localOutRules := LocalSecurityRuleSet{}
-	for i := range localOutAllowList {
-		rule := LocalSecurityRule{}
-		rule.SecurityRule = localOutAllowList[i]
-		_localOutRules = append(_localOutRules, rule)
-	}
-	if onlyAllowRules {
-		localOutRules, localInRules = _localOutRules, _localInRules
-	}
-	if len(_localInRules) < len(localInRules) {
-		localInRules = _localInRules
-	}
-	if len(_localOutRules) < len(localOutRules) {
-		localOutRules = _localOutRules
-	}
+	SortSecurityRule(destInRules, dest.MaxPriority, dest.MinPriority, false, dest.IsOnlySupportAllowRules)
+	SortSecurityRule(destOutRules, dest.MaxPriority, dest.MinPriority, false, dest.IsOnlySupportAllowRules)
 
-	SortSecurityRule(inRules, order, onlyAllowRules)
-	SortSecurityRule(outRules, order, onlyAllowRules)
+	srcInAllowList := srcInRules.AllowList()
+	srcOutAllowList := srcOutRules.AllowList()
+
+	destInAllowList := destInRules.AllowList()
+	destOutAllowList := destOutRules.AllowList()
+	inEquals, outEquals := isAllowListEqual(srcInAllowList, destInAllowList), isAllowListEqual(srcOutAllowList, destOutAllowList)
 
-	inAllowList := inRules.AllowList()
-	outAllowList := outRules.AllowList()
-	inEquals, outEquals := inAllowList.Equals(localInAllowList), outAllowList.Equals(localOutAllowList)
 	if inEquals && outEquals {
 		return
 	}
 
-	// priority从小到大排列(从默认规则开始对比)
-	sort.Sort(sort.Reverse(localInRules))
-	sort.Sort(sort.Reverse(localOutRules))
-
-	sort.Sort(sort.Reverse(inRules))
-	sort.Sort(sort.Reverse(outRules))
-
-	startPriority := minPriority - 1
-	if order == PriorityOrderByAsc {
-		startPriority = maxPriority + 1
+	if debug {
+		log.Debugf("In: src: %s dest: %s result: %v", srcInAllowList.String(), destInAllowList.String(), inEquals)
+		log.Debugf("Out: src: %s dest: %s result: %v", srcOutAllowList.String(), destOutAllowList.String(), outEquals)
 	}
 
-	var addPriority = func(priority int, order TPriorityOrder, inc int, min, max int, onlyAllowRules bool) int {
+	var tryUseAllowList = func(defaultRule SecurityRule, allowList secrules.SecurityRuleSet, rules SecurityRuleSet, isOnlyAllowList bool) SecurityRuleSet {
+		if len(allowList) < len(rules) || isOnlyAllowList {
+			rules = SecurityRuleSet{}
+			for i := range allowList {
+				rule := SecurityRule{}
+				rule.SecurityRule = allowList[i]
+				rules = append(rules, rule)
+			}
+
+			if !utils.IsInStringArray(allowList.String(), []string{
+				"",
+				"in:allow any",
+				"out:allow any",
+				"in:deny any",
+				"out:deny any",
+			}) && strings.HasSuffix(defaultRule.SecurityRule.String(), "deny any") {
+				rules = append(rules, defaultRule)
+			}
+		}
+		return rules
+	}
+
+	srcInRules = tryUseAllowList(src.InDefaultRule, srcInAllowList, srcInRules, dest.IsOnlySupportAllowRules)
+	srcOutRules = tryUseAllowList(src.OutDefaultRule, srcOutAllowList, srcOutRules, dest.IsOnlySupportAllowRules)
+
+	if inEquals {
+		srcInRules, destInRules = []SecurityRule{}, []SecurityRule{}
+	}
+	if outEquals {
+		srcOutRules, destOutRules = []SecurityRule{}, []SecurityRule{}
+	}
+
+	if debug {
+		log.Debugf("src in rules: ")
+		srcInRules.Debug()
+	}
+
+	// 默认从优先级低到高比较
+	SortSecurityRule(srcInRules, src.MaxPriority, src.MinPriority, true, src.IsOnlySupportAllowRules)
+	SortSecurityRule(srcOutRules, src.MaxPriority, src.MinPriority, true, src.IsOnlySupportAllowRules)
+
+	SortSecurityRule(destInRules, dest.MaxPriority, dest.MinPriority, true, dest.IsOnlySupportAllowRules)
+	SortSecurityRule(destOutRules, dest.MaxPriority, dest.MinPriority, true, dest.IsOnlySupportAllowRules)
+
+	var addPriority = func(priority int, min, max int, onlyAllowRules bool) int {
 		if onlyAllowRules {
-			return 0
+			return priority
 		}
-		inc = inc * int(order) //+ int(order)
-		priority += inc
-		if priority < min {
-			return min
+		inc := 1
+		if max < min {
+			max, min, inc = min, max, -1
 		}
-		if priority > max {
-			return max
+		if priority >= max || priority <= min {
+			return priority
 		}
-		return priority
+		return priority + inc
 	}
 
-	var getInitPriority = func(init, min, max int) int {
-		if init < min || init > max {
-			return (min + max) / 2
-		}
-		return init
-	}
-
-	var compare = func(localRules LocalSecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
-		i, j, inc, prePriority := 0, 0, 1, 0
-		for i < len(localRules) || j < len(remoteRules) {
-			if i < len(localRules) && j < len(remoteRules) {
-				ruleStr := remoteRules[j].String()
-				localRuleStr := localRules[i].String()
+	var _compare = func(srcRules SecurityRuleSet, destRules SecurityRuleSet) (common, add, del SecurityRuleSet) {
+		i, j, priority := 0, 0, (dest.MinPriority-1+dest.MaxPriority)/2
+		for i < len(srcRules) || j < len(destRules) {
+			if i < len(srcRules) && j < len(destRules) {
+				destRuleStr := destRules[j].String()
+				srcRuleStr := srcRules[i].String()
 				if debug {
-					log.Debugf("compare local priority(%d) %s -> remote name(%s) priority(%d) %s\n", localRules[i].Priority, localRules[i].String(), remoteRules[j].Name, remoteRules[j].Priority, remoteRules[j].String())
+					log.Debugf("compare src %s(%s) priority(%d) %s -> dest name(%s) %s(%s) priority(%d) %s\n",
+						srcRules[i].Id, srcRules[i].ExternalId, srcRules[i].Priority, srcRules[i].String(),
+						destRules[j].Name, destRules[j].ExternalId, destRules[j].Id, destRules[j].Priority, destRules[j].String())
 				}
-				cmp := strings.Compare(ruleStr, localRuleStr)
+				cmp := strings.Compare(destRuleStr, srcRuleStr)
 				if cmp == 0 {
-					prePriority = remoteRules[j].Priority
-					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
-						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
-					}
-					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-						common = append(common, remoteRules[j])
+					destRules[j].Id = srcRules[i].Id
+					common = append(common, destRules[j])
+					if destRules[j].ExternalId != DEFAULT_DEST_RULE_ID {
+						priority = destRules[j].Priority
 					}
 					i++
 					j++
 				} else if cmp < 0 {
-					if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
-						del = append(del, remoteRules[j])
-					}
+					del = append(del, destRules[j])
 					j++
 				} else {
-					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
-					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-						add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
-					}
+					priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+					srcRules[i].Priority = priority
+					add = append(add, srcRules[i])
 					i++
-					inc++
-				}
-			} else if i >= len(localRules) {
-				if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
-					del = append(del, remoteRules[j])
 				}
+			} else if i >= len(srcRules) {
+				del = append(del, destRules[j])
 				j++
-			} else if j >= len(remoteRules) {
-				initPriority := startPriority
-				if len(remoteRules) > 0 {
-					initPriority = remoteRules[len(remoteRules)-1].Priority
-				}
-				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
-				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-				if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-					(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-					(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-					add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
-				}
+			} else if j >= len(destRules) {
+				priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+				srcRules[i].Priority = priority
+				add = append(add, srcRules[i])
 				i++
-				inc++
 			}
 		}
 		return
 	}
 
 	type rulePair struct {
-		localRules  LocalSecurityRuleSet
-		remoteRules []SecurityRule
-		protocol    string
+		srcRules  SecurityRuleSet
+		destRules SecurityRuleSet
+		protocol  string
 	}
 
-	var splitRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) []rulePair {
+	var splitRules = func(src, dest SecurityRuleSet) []rulePair {
 		rules := map[string]rulePair{}
-		for _, r := range localRules {
+		for _, r := range src {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{srcRules: SecurityRuleSet{}, destRules: SecurityRuleSet{}, protocol: r.Protocol}
 			}
-			pair.localRules = append(pair.localRules, r)
+			pair.srcRules = append(pair.srcRules, r)
 			rules[r.Protocol] = pair
 		}
 
-		for _, r := range remoteRules {
+		for _, r := range dest {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{srcRules: SecurityRuleSet{}, destRules: SecurityRuleSet{}, protocol: r.Protocol}
 			}
-			pair.remoteRules = append(pair.remoteRules, r)
+			pair.destRules = append(pair.destRules, r)
 			rules[r.Protocol] = pair
 		}
 
@@ -352,23 +330,61 @@ func CompareRules(
 		return ret
 	}
 
-	var compareRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
-		pairs := splitRules(localRules, remoteRules)
+	var compare = func(src, dest SecurityRuleSet) (common, added, dels SecurityRuleSet) {
+		pairs := splitRules(src, dest)
 		for _, r := range pairs {
-			_common, _add, _dels := compare(r.localRules, r.remoteRules)
+			_common, _add, _dels := _compare(r.srcRules, r.destRules)
 			common = append(common, _common...)
-			add = append(add, _add...)
+			added = append(added, _add...)
 			dels = append(dels, _dels...)
 		}
 		return
 	}
 
-	if !inEquals {
-		inCommon, inAdds, inDels = compareRules(localInRules, inRules)
+	var inCommon, outCommon SecurityRuleSet
+	inCommon, inAdds, inDels = compare(srcInRules, destInRules)
+	outCommon, outAdds, outDels = compare(srcOutRules, destOutRules)
+
+	var handleDefaultRules = func(removed, added []SecurityRule, isOnlyAllowList bool) ([]SecurityRule, []SecurityRule) {
+		ret := []SecurityRule{}
+		for _, rule := range removed {
+			if rule.ExternalId == DEFAULT_DEST_RULE_ID {
+				if debug {
+					log.Debugf("remove dest default rule: %s external id %s priority: %d", rule.String(), rule.ExternalId, rule.Priority)
+				}
+				if rule.Action == secrules.SecurityRuleDeny && isOnlyAllowList {
+					continue
+				}
+				switch rule.Action {
+				case secrules.SecurityRuleDeny:
+					rule.Action = secrules.SecurityRuleAllow
+				case secrules.SecurityRuleAllow:
+					rule.Action = secrules.SecurityRuleDeny
+				}
+				rule.Priority = dest.MinPriority
+
+				find := false
+				for i := range added {
+					if added[i].String() == rule.String() {
+						find = true
+						break
+					}
+				}
+				if !find {
+					if debug {
+						log.Debugf("add new default rule: %s external id %s priority: %d", rule.String(), rule.ExternalId, rule.Priority)
+					}
+					added = append(added, rule)
+				}
+			} else {
+				ret = append(ret, rule)
+			}
+		}
+		return ret, added
 	}
-	if !outEquals {
-		outCommon, outAdds, outDels = compareRules(localOutRules, outRules)
-	}
-	common = append(inCommon, outCommon...)
+
+	inDels, inAdds = handleDefaultRules(inDels, inAdds, dest.IsOnlySupportAllowRules)
+	outDels, outAdds = handleDefaultRules(outDels, outAdds, dest.IsOnlySupportAllowRules)
+	common, _ = handleDefaultRules(append(inCommon, outCommon...), []SecurityRule{}, dest.IsOnlySupportAllowRules)
 	return
 }

pkg/compute/guestdrivers/aliyun.go

@@ -164,6 +164,39 @@ func (self *SAliyunGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_READY
 }
 
+func (self *SAliyunGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_EFFICIENCY, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD_PL2, MaxSizeGb: 32768, MinSizeGb: 461, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD_PL3, MaxSizeGb: 32768, MinSizeGb: 1261, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PUBLIC_CLOUD, MaxSizeGb: 2000, MinSizeGb: 5, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_EPHEMERAL_SSD, MaxSizeGb: 800, MinSizeGb: 5, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_EFFICIENCY, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+			},
+		},
+	}
+}
+
 func (self *SAliyunGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/aws.go

@@ -86,6 +86,39 @@ func (self *SAwsGuestDriver) GetWindowsUserDataType() string {
 	return cloudprovider.CLOUD_EC2
 }
 
+func (self *SAwsGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SAwsGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	// return fetchAwsUserName(desc)
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/azure.go

@@ -222,6 +222,35 @@ func (self *SAzureGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_READY
 }
 
+func (self *SAzureGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_AZURE_DEFAULT_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARDSSD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PREMIUM_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARDSSD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PREMIUM_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SAzureGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	return api.VM_AZURE_DEFAULT_LOGIN_USER
 }

pkg/compute/guestdrivers/baremetals.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
@@ -56,6 +57,13 @@ func (self *SBaremetalGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_ONECLOUD
 }
 
+func (self *SBaremetalGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+	}
+}
+
 func (self *SBaremetalGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/base.go

@@ -398,3 +398,7 @@ func (self *SBaseGuestDriver) RequestRemoteUpdate(ctx context.Context, guest *mo
 	// nil ops
 	return nil
 }
+
+func (self *SBaseGuestDriver) ValidateDetachNetwork(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
+	return nil
+}

pkg/compute/guestdrivers/container.go

@@ -24,6 +24,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -57,6 +58,13 @@ func (self *SContainerDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_ONECLOUD
 }
 
+func (self *SContainerDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+	}
+}
+
 // for backward compatibility, deprecated driver
 func (self *SContainerDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}

pkg/compute/guestdrivers/ctyun.go

@@ -39,6 +39,23 @@ func (self *SCtyunGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_CTYUN
 }
 
+func (self *SCtyunGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SCtyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/esxi.go

@@ -65,6 +65,23 @@ func (self *SESXiGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_VMWARE
 }
 
+func (self *SESXiGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     true,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SESXiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/google.go

@@ -65,6 +65,33 @@ func (self *SGoogleGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_GOOGLE
 }
 
+func (self *SGoogleGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_SSD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_STANDARD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_BALANCED, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_SSD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_STANDARD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_BALANCED, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SGoogleGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_GOOGLE_PD_STANDARD
 }

pkg/compute/guestdrivers/huawei.go

@@ -110,6 +110,35 @@ func (self *SHuaweiGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_RUNNING
 }
 
+func (self *SHuaweiGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SHuaweiGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/kvm.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -68,6 +69,22 @@ func (self *SKVMGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, own
 	return keys
 }
 
+func (self *SKVMGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     true,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+	}
+}
+
 func (self *SKVMGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_LOCAL
 }
@@ -629,3 +646,10 @@ func (self *SKVMGuestDriver) CheckLiveMigrate(guest *models.SGuest, userCred mcc
 	}
 	return nil
 }
+
+func (self *SKVMGuestDriver) ValidateDetachNetwork(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
+	if guest.Status == api.VM_RUNNING && guest.GetMetadata("hot_remove_nic", nil) != "enable" {
+		return httperrors.NewBadRequestError("Guest %s can't hot remove nic", guest.GetName())
+	}
+	return nil
+}

pkg/compute/guestdrivers/managedvirtual.go

@@ -649,8 +649,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte
 			if errors.Cause(err) == cloudprovider.ErrNotFound {
 				return nil, nil
 			}
-			log.Errorf("host.GetIHost fail %s", err)
-			return nil, err
+			return nil, errors.Wrapf(err, "host.GetIHost")
 		}
 
 		// 创建失败时external id为空。此时直接返回即可。不需要再调用公有云api
@@ -663,14 +662,11 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte
 			if errors.Cause(err) == cloudprovider.ErrNotFound {
 				return nil, nil
 			}
-
-			log.Errorf("ihost.GetIVMById fail %s", err)
-			return nil, err
+			return nil, errors.Wrapf(err, "ihost.GetIVMById(%s)", guest.ExternalId)
 		}
 		err = ivm.DeleteVM(ctx)
 		if err != nil {
-			log.Errorf("ivm.DeleteVM fail %s", err)
-			return nil, err
+			return nil, errors.Wrapf(err, "ivm.DeleteVM")
 		}
 
 		for _, guestdisk := range guest.GetDisks() {
@@ -682,16 +678,14 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte
 					if errors.Cause(err) == cloudprovider.ErrNotFound {
 						continue
 					}
-					log.Errorf("disk.GetIDisk fail %s", err)
-					return nil, err
+					return nil, errors.Wrapf(err, "disk.GetIDisk")
 				}
 				if idisk.GetStatus() == api.DISK_DEALLOC {
 					continue
 				}
 				err = idisk.Delete(ctx)
 				if err != nil {
-					log.Errorf("idisk.Delete fail %s", err)
-					return nil, err
+					return nil, errors.Wrapf(err, "idisk.Delete")
 				}
 			}
 		}

pkg/compute/guestdrivers/openstack.go

@@ -64,6 +64,23 @@ func (self *SOpenStackGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_OPENSTACK
 }
 
+func (self *SOpenStackGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SOpenStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/qcloud.go

@@ -136,7 +136,7 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		if sysDisk.SizeMb > 1024*1024 {
 			return nil, fmt.Errorf("The %s system disk size must be less than 1024GB", sysDisk.Backend)
 		}
-	case api.STORAGE_LOCAL_PRO:
+	case api.STORAGE_LOCAL_PRO, api.STORAGE_CLOUD_HSSD: //https://cloud.tencent.com/document/product/362/2353
 		return nil, fmt.Errorf("storage %s can not be system disk", sysDisk.Backend)
 	}
 
@@ -243,6 +243,43 @@ func (self *SQcloudGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }
 
+func (self *SQcloudGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+				DistAccounts: []cloudprovider.SDistDefaultAccount{
+					{
+						OsDistribution: "Ubuntu",
+						DefaultAccount: "ubuntu",
+						Changeable:     false,
+					},
+				},
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_BASIC, MaxSizeGb: 16000, MinSizeGb: 10, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_PREMIUM, MaxSizeGb: 16000, MinSizeGb: 50, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 16000, MinSizeGb: 100, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_HSSD, MaxSizeGb: 32000, MinSizeGb: 20, StepSizeGb: 10, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_BASIC, MaxSizeGb: 500, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_PREMIUM, MaxSizeGb: 1024, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 500, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SQcloudGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.ImageType == "system" {

pkg/compute/guestdrivers/ucloud.go

@@ -96,6 +96,21 @@ func (self *SUCloudGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	return nil
 }
 
+func (self *SUCloudGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+	}
+}
+
 func (self *SUCloudGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	if desc.OsType == "Windows" {
 		return "Administrator"