Merge pull request #11410 from yousong/automated-cherry-pick-of-#11406-upstream-release-3.4

Automated cherry pick of #11406: webconsole: wait zombies

Makefile

@@ -9,7 +9,7 @@ BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
 
 ifeq ($(ONECLOUD_CI_BUILD),)
 	GIT_COMMIT := $(shell git rev-parse --short HEAD)
-	GIT_BRANCH := $(shell git name-rev --name-only HEAD)
+	GIT_BRANCH := $(shell git branch -r --contains | head -1 | sed -E -e "s%(HEAD ->|origin|upstream)/?%%g" | xargs)
 	GIT_VERSION := $(shell git describe --always --tags --abbrev=14 $(GIT_COMMIT)^{commit})
 	GIT_TREE_STATE := $(shell s=`git status --porcelain 2>/dev/null`; if [ -z "$$s" ]; then echo "clean"; else echo "dirty"; fi)
 	BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')

cmd/ansibleserver/main.go

@@ -15,9 +15,14 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/ansibleserver/service"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/baremetal-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/baremetal/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/climc/shell/compute/cloudregions.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -26,6 +27,23 @@ import (
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.Cloudregions).WithKeyword("cloud-region")
+	cmd.PerformClass("sync-skus", &options.CloudregionSkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "cloud-region-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.Cloudregions.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudregionListOptions struct {
 		options.BaseListOptions
 

cmd/climc/shell/compute/dbinstance_skus.go

@@ -15,12 +15,30 @@
 package compute
 
 import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.DBInstanceSkus).WithKeyword("dbinstance-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "dbinstance-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.DBInstanceSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type DBInstanceSkuListOption struct {
 		options.BaseListOptions
 		Engine        string
@@ -54,5 +72,4 @@ func init() {
 		printObject(result)
 		return nil
 	})
-
 }

cmd/climc/shell/compute/elasticcache.go

@@ -19,12 +19,30 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ElasticcacheSkus).WithKeyword("elastic-cache-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "elastic-cache-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ElasticcacheSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.BaseListOptions{}, "elastic-cache-list", "List elastisc cache instance", func(s *mcclient.ClientSession, opts *options.BaseListOptions) error {
 		params, err := options.ListStructToParams(opts)
 		if err != nil {

cmd/climc/shell/compute/elasticips.go

@@ -50,7 +50,7 @@ func init() {
 		Manager    string `help:"cloud provider"`
 		Region     string `help:"cloud region in which EIP is allocated"`
 		Bandwidth  int    `help:"Bandwidth in Mbps"`
-		Ip         string `help:"IP address of the EIP"`
+		IpAddr     string `help:"IP address of the EIP" json:"ip_addr"`
 		Network    string `help:"Network of the EIP"`
 		ChargeType string `help:"bandwidth charge type" choices:"traffic|bandwidth"`
 	}
@@ -75,8 +75,8 @@ func init() {
 			params.Add(jsonutils.NewString(args.Network), "network")
 		}
 
-		if len(args.Ip) > 0 {
-			params.Add(jsonutils.NewString(args.Ip), "ip")
+		if len(args.IpAddr) > 0 {
+			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
 
 		result, err := modules.Elasticips.Create(s, params)

cmd/climc/shell/compute/hosts.go

@@ -407,12 +407,14 @@ func init() {
 	})
 
 	type HostAddNetIfOptions struct {
-		ID     string `help:"ID or Name of host"`
-		WIRE   string `help:"ID or Name of wire to attach"`
-		MAC    string `help:"Mac address of NIC"`
-		INDEX  int64  `help:"nic index"`
-		Type   string `help:"Nic type" choices:"admin|ipmi"`
-		IpAddr string `help:"IP address"`
+		ID        string `help:"ID or Name of host"`
+		WIRE      string `help:"ID or Name of wire to attach"`
+		MAC       string `help:"Mac address of NIC"`
+		INDEX     int64  `help:"nic index"`
+		Type      string `help:"Nic type" choices:"admin|ipmi"`
+		IpAddr    string `help:"IP address"`
+		Bridge    string `help:"Bridge of hostwire"`
+		Interface string `help:"Interface name, eg:eth0, en0"`
 	}
 	R(&HostAddNetIfOptions{}, "host-add-netif", "Host add a NIC", func(s *mcclient.ClientSession, args *HostAddNetIfOptions) error {
 		params := jsonutils.NewDict()
@@ -426,6 +428,12 @@ func init() {
 		if len(args.IpAddr) > 0 {
 			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
+		if len(args.Bridge) > 0 {
+			params.Add(jsonutils.NewString(args.Bridge), "bridge")
+		}
+		if len(args.Interface) > 0 {
+			params.Add(jsonutils.NewString(args.Interface), "interface")
+		}
 		result, err := modules.Hosts.PerformAction(s, args.ID, "add-netif", params)
 		if err != nil {
 			return err

cmd/climc/shell/compute/quotas.go

@@ -316,7 +316,7 @@ func init() {
 		QuotaSetBaseOptions
 		IdentityQuotaOptions
 	}
-	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *InfrasQuotaSetOptions) error {
+	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *IdentityQuotaSetOptions) error {
 		params := jsonutils.Marshal(args)
 		quotas, e := modules.IdentityQuotas.DoQuotaSet(s, params)
 		if e != nil {

cmd/climc/shell/compute/schedulers.go

@@ -123,9 +123,10 @@ func init() {
 		})
 
 	type SchedulerHistoryListOptions struct {
-		Limit  int  `default:"50" help:"Page limit"`
-		Offset int  `default:"0" help:"Page offset"`
-		All    bool `help:"Show all histories, including scheduler-test"`
+		Limit        int  `default:"50" help:"Page limit"`
+		Offset       int  `default:"0" help:"Page offset"`
+		All          bool `help:"Show all histories, including scheduler-test"`
+		IsSuggestion bool `help:"Only show forcast suggestion history"`
 	}
 	R(&SchedulerHistoryListOptions{}, "scheduler-history-list", "Show scheduler history list",
 		func(s *mcclient.ClientSession, args *SchedulerHistoryListOptions) error {
@@ -138,17 +139,14 @@ func init() {
 			if args.Offset > 0 {
 				params.Add(jsonutils.NewInt(int64(args.Offset)), "offset")
 			}
-			if args.All {
-				params.Add(jsonutils.JSONTrue, "all")
-			} else {
-				params.Add(jsonutils.JSONFalse, "all")
-			}
+			params.Add(jsonutils.NewBool(args.All), "all")
+			params.Add(jsonutils.NewBool(args.IsSuggestion), "is_suggestion")
 			result, err := modules.SchedManager.HistoryList(s, params)
 			if err != nil {
 				return err
 			}
 			printList(modulebase.JSON2ListResult(result), []string{
-				"session_id", "time", "status", "consuming",
+				"session_id", "time", "status", "consuming", "is_suggestion",
 			})
 			return nil
 		})

cmd/climc/shell/compute/skus.go

@@ -17,12 +17,30 @@ package compute
 import (
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ServerSkus).WithKeyword("server-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "server-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ServerSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type ServerSkusListOptions struct {
 		options.BaseListOptions
 		Cloudregion    string  `help:"region Id or name"`

cmd/climc/shell/compute/storages.go

@@ -61,13 +61,14 @@ func init() {
 		ID                    string  `help:"ID or Name of storage to update"`
 		Name                  string  `help:"New Name of storage"`
 		Desc                  string  `help:"Description"`
-		CommitBound           float64 `help:"Upper bound of storage overcommit rate"`
+		CommitBound           float64 `help:"Upper bound of storage overcommit rate" json:"cmtbound"`
 		MediumType            string  `help:"Medium type" choices:"ssd|rotate"`
 		RbdRadosMonOpTimeout  int64   `help:"ceph rados_mon_op_timeout"`
 		RbdRadosOsdOpTimeout  int64   `help:"ceph rados_osd_op_timeout"`
 		RbdClientMountTimeout int64   `help:"ceph client_mount_timeout"`
 		RbdKey                string  `help:"ceph rbd key"`
 		Reserved              string  `help:"Reserved storage space"`
+		Capacity              int     `help:"Capacity for storage"`
 	}
 	R(&StorageUpdateOptions{}, "storage-update", "Update a storage", func(s *mcclient.ClientSession, args *StorageUpdateOptions) error {
 		params, err := options.StructToParams(args)

cmd/climc/shell/compute/webconsole.go

@@ -42,34 +42,24 @@ func init() {
 		if err != nil {
 			return err
 		}
-		connParams, err := obj.GetString("connect_params")
-		if err != nil {
+
+		resp := &webconsole_api.ServerRemoteConsoleResponse{}
+		if err := obj.Unmarshal(resp); err != nil {
 			return err
 		}
-		if decodeStr, err := base64.StdEncoding.DecodeString(connParams); err == nil {
-			connParams = string(decodeStr)
-		}
-		var query url.Values
-		connUrl, err := url.ParseRequestURI(connParams)
-		if err == nil {
-			query = connUrl.Query()
-		} else {
-			query, err = url.ParseQuery(connParams)
-			if err != nil {
-				return err
-			}
-		}
-		protocol := query.Get("protocol")
-		if !utils.IsInStringArray(protocol, []string{
+		connectParams := resp.GetConnectParams()
+		if protocol, err := resp.GetConnectProtocol(); err != nil {
+			return err
+		} else if !utils.IsInStringArray(protocol, []string{
 			command.PROTOCOL_TTY, webconsole_api.VNC,
 			webconsole_api.SPICE, webconsole_api.WMKS,
 		}) {
-			fmt.Println(connParams)
+			fmt.Println(connectParams)
 			return nil
 		}
 
-		newQuery := url.Values(make(map[string][]string))
-		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connParams)))
+		newQuery := url.Values{}
+		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connectParams)))
 		u.RawQuery = newQuery.Encode()
 		fmt.Println(u.String())
 		return nil

cmd/climc/shell/helper.go

@@ -284,7 +284,10 @@ type IPerformOpt interface {
 func (cmd ResourceCmd) Perform(action string, args IPerformOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IPerformOpt) error {
-		params := jsonutils.Marshal(args) // .Params()
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
 		ret, err := man.(modulebase.Manager).PerformAction(s, args.GetId(), action, params)
 		if err != nil {
 			return err

cmd/climc/shell/identity/policies.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -183,6 +184,7 @@ func init() {
 	type PolicyShowOptions struct {
 		ID     string `help:"ID of policy"`
 		Format string `help:"policy format, default yaml" default:"yaml" choices:"yaml|json"`
+		Save   string `help:"save policy data into a file"`
 	}
 	R(&PolicyShowOptions{}, "policy-show", "Show policy", func(s *mcclient.ClientSession, args *PolicyShowOptions) error {
 		query := jsonutils.NewDict()
@@ -192,6 +194,15 @@ func init() {
 			return err
 		}
 		printObject(result)
+		if len(args.Save) > 0 {
+			if args.Format == "yaml" {
+				yaml, _ := result.GetString("policy")
+				fileutils2.FilePutContents(args.Save, yaml, false)
+			} else {
+				p, _ := result.Get("policy")
+				fileutils2.FilePutContents(args.Save, p.PrettyString(), false)
+			}
+		}
 		return nil
 	})
 

cmd/climc/shell/misc/copyright.go

@@ -36,6 +36,8 @@ func init() {
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
 		License   *string `help:"the license generator website address"`
+		BrandCn   *string `help:"the brand name in Chinese"`
+		BrandEn   *string `help:"the brand name in English"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -52,6 +54,14 @@ func init() {
 			params.Add(jsonutils.NewString(*args.License), "license")
 		}
 
+		if args.BrandCn != nil {
+			params.Add(jsonutils.NewString(*args.BrandCn), "brand_cn")
+		}
+
+		if args.BrandEn != nil {
+			params.Add(jsonutils.NewString(*args.BrandEn), "brand_en")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

cmd/esxi-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/esxi/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/glance/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/image/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/host-deployer/main.go

@@ -14,9 +14,16 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	deployer := deployserver.NewDeployService()
 	deployer.StartService()
 }

cmd/host/main.go

@@ -15,14 +15,19 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/service"
 	"yunion.io/x/onecloud/pkg/hostman"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	var srv = &hostman.SHostService{}
 	srv.SServiceBase = &service.SServiceBase{
 		Service: srv,

cmd/vpcagent/main.go

@@ -26,6 +26,7 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/vpcagent/options"
 	_ "yunion.io/x/onecloud/pkg/vpcagent/ovn"
 	"yunion.io/x/onecloud/pkg/vpcagent/worker"
@@ -52,8 +53,12 @@ func main() {
 	}
 
 	{
+
+		ctx := context.Background()
+		ctx, cancelFunc := context.WithCancel(ctx)
+		go procutils.WaitZombieLoop(ctx)
+
 		wg := &sync.WaitGroup{}
-		ctx, cancelFunc := context.WithCancel(context.Background())
 		ctx = context.WithValue(ctx, "wg", wg)
 		wg.Add(1)
 		go w.Start(ctx)

cmd/webconsole/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

pkg/apigateway/handler/auth.go

@@ -156,6 +156,9 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 			}
 		}
 		resp.Add(domains, "domains")
+		resp.Add(jsonutils.JSONTrue, "return_full_domains")
+	} else {
+		resp.Add(jsonutils.JSONFalse, "return_full_domains")
 	}
 
 	filters := jsonutils.NewDict()
@@ -962,6 +965,12 @@ func getUserInfo2(s *mcclient.ClientSession, uid string, pid string, loginIp str
 		data.Add(jsonutils.JSONFalse, "non_default_domain_projects")
 	}
 
+	if options.Options.EnableQuotaCheck {
+		data.Add(jsonutils.JSONTrue, "enable_quota_check")
+	} else {
+		data.Add(jsonutils.JSONFalse, "enable_quota_check")
+	}
+
 	data.Add(jsonutils.NewString(getSsoCallbackUrl()), "sso_callback_url")
 
 	return data, nil

pkg/apis/compute/cloudregions_const.go

@@ -114,6 +114,7 @@ const (
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
 	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
+	CITY_PHOENIX        = "Phoenix"        //菲尼克斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -150,7 +151,8 @@ const (
 	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
 
 	// 巴西
-	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+	CITY_SAO_PAULO      = "Sao Paulo"      //圣保罗
+	CITY_RIO_DE_JANEIRO = "Rio de Janeiro" // 里约热内卢
 
 	// 智利
 	CITY_SANTIAGO = "Santiago" // 圣地亚哥

pkg/apis/compute/elasticip.go

@@ -67,6 +67,8 @@ type SElasticipCreateInput struct {
 	// 子网Id
 	// 私有云创建此参数必传,例如Openstack, ZStack
 	NetworkId string `json:"network_id"`
+	// 指定新建EIP的地址
+	IpAddr string `json:"ip_addr"`
 }
 
 type ElasticipDetails struct {

pkg/apis/compute/elasticips_const.go

@@ -42,7 +42,6 @@ const (
 
 	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
 	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
-	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
 )
 
 var (

pkg/apis/compute/guest_const.go

@@ -249,6 +249,8 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 }
 
 const (
+	VM_DEFAULT_WINDOWS_LOGIN_USER     = "Administrator"
+	VM_DEFAULT_LINUX_LOGIN_USER       = "root"
 	VM_AWS_DEFAULT_LOGIN_USER         = "ec2user"
 	VM_AWS_DEFAULT_WINDOWS_LOGIN_USER = "Administrator"
 	VM_AZURE_DEFAULT_LOGIN_USER       = "toor"

pkg/apis/compute/guests.go

@@ -214,6 +214,8 @@ type ServerDetails struct {
 	Vpc string `json:"vpc"`
 	// 归属VPC ID
 	VpcId string `json:"vpc_id"`
+	// Vpc外网访问模式
+	VpcExternalAccessMode string `json:"vpc_external_access_mode"`
 
 	// 关联安全组列表
 	Secgroups []apis.StandaloneShortDesc `json:"secgroups"`

pkg/apis/compute/network_const.go

@@ -62,6 +62,7 @@ var (
 		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,
+		CLOUD_PROVIDER_OPENSTACK,
 	}
 )
 

pkg/apis/compute/sku_const.go

@@ -155,3 +155,25 @@ type DBInstanceSkuListInput struct {
 	Zone2 []string `json:"zone2"`
 	Zone3 []string `json:"zone3"`
 }
+
+type SkuSyncInput struct {
+	// 云平台名称
+	// example: Google
+	Provider string `json:"provider,omitempty"`
+
+	// 区域ID
+	CloudregionIds []string `json:"cloudregion_ids"`
+}
+
+type SkuTaskQueryInput struct {
+	// 异步任务ID
+	TaskIds []string `json:"task_ids"`
+}
+
+type CloudregionSkuSyncInput struct {
+	SkuSyncInput
+
+	// 同步资源类型
+	// choices: serversku|elasticcachesku|dbinstance_sku
+	Resource string `json:"resource"`
+}

pkg/apis/compute/storage_const.go

@@ -59,9 +59,10 @@ const (
 	STORAGE_CLOUD_HSSD    = "cloud_hssd"    //增强型SSD云硬盘
 
 	// huawei storage type
-	STORAGE_HUAWEI_SSD  = "SSD"  // 超高IO云硬盘
-	STORAGE_HUAWEI_SAS  = "SAS"  // 高IO云硬盘
-	STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘
+	STORAGE_HUAWEI_SSD   = "SSD"   // 超高IO云硬盘
+	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
+	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
+	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/compute/vpcs_ovn.go

@@ -18,10 +18,13 @@ import (
 	"yunion.io/x/pkg/util/netutils"
 )
 
+const VPC_OVN_ENCAP_COST = 58
+
 const (
 	VPC_EXTERNAL_ACCESS_MODE_DISTGW     = "distgw"     // distgw only
 	VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW = "eip-distgw" // eip when available, distgw otherwise
 	VPC_EXTERNAL_ACCESS_MODE_EIP        = "eip"        // eip only
+	VPC_EXTERNAL_ACCESS_MODE_NONE       = "none"       // no external access
 )
 
 var (
@@ -29,6 +32,7 @@ var (
 		VPC_EXTERNAL_ACCESS_MODE_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP,
+		VPC_EXTERNAL_ACCESS_MODE_NONE,
 	}
 )
 

pkg/apis/identity/consts.go

@@ -99,6 +99,7 @@ var (
 
 	CommonWhitelistOptionMap = map[string][]string{
 		"default": []string{
+			"enable_quota_check",
 			"default_quota_value",
 			"enable_rbac",
 			"non_default_domain_projects",

pkg/apis/input.go

@@ -201,6 +201,8 @@ type PerformStatusInput struct {
 	// 更改的目标状态值
 	// required:true
 	Status string `json:"status"`
+	// swagger:ignore
+	BlockJobsCount int `json:"block_jobs_count"`
 
 	// 更改状态的原因描述
 	// required:false

pkg/apis/monitor/alert.go

@@ -117,6 +117,7 @@ type AlertCreateInput struct {
 	NoDataState string `json:"no_data_state"`
 	// 报警执行错误将当前报警状态设置为对应的状态
 	ExecutionErrorState string `json:"execution_error_state"`
+	UsedBy              string `json:"used_by"`
 }
 
 type AlertUpdateInput struct {

pkg/apis/webconsole/types.go

@@ -0,0 +1,53 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package webconsole
+
+import (
+	"encoding/base64"
+	"net/url"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type ServerRemoteConsoleResponse struct {
+	ConnectParams string `json:"connect_params"`
+	Session       string `json:"session,omitempty"`
+
+	apis.Meta
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectParams() string {
+	params := resp.ConnectParams
+	if data, err := base64.StdEncoding.DecodeString(params); err == nil {
+		params = string(data)
+	}
+	return params
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectProtocol() (string, error) {
+	var (
+		params = resp.GetConnectParams()
+		query  url.Values
+	)
+	if uri, err := url.ParseRequestURI(params); err == nil {
+		query = uri.Query()
+	} else if q, err := url.ParseQuery(params); err == nil {
+		query = q
+	} else {
+		return "", err
+	}
+	protocol := query.Get("protocol")
+	return protocol, nil
+}

pkg/baremetal/cronjobs.go

@@ -93,7 +93,7 @@ func (job *SStatusProbeJob) Name() string {
 
 func (job *SStatusProbeJob) Do(ctx context.Context, now time.Time) error {
 	bStatus := job.baremetal.GetStatus()
-	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING {
+	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING || bStatus == api.BAREMETAL_UNKNOWN {
 		ps, err := job.baremetal.GetPowerStatus()
 		if err != nil {
 			return errors.Wrap(err, "GetPowerStatus")

pkg/baremetal/pxe/dhcp.go

@@ -368,7 +368,7 @@ func (req *dhcpRequest) createBaremetal(session *mcclient.ClientSession) (jsonut
 	params := jsonutils.NewDict()
 	mac := req.ClientMac.String()
 	zoneId := req.baremetalManager.GetZoneId()
-	name := fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1))
+	name := strings.ToLower(fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1)))
 	params.Add(jsonutils.NewString(name), "name")
 	params.Add(jsonutils.NewString(mac), "access_mac")
 	params.Add(jsonutils.NewString("baremetal"), "host_type")

pkg/baremetal/utils/ipmitool/ipmitool.go

@@ -15,6 +15,7 @@
 package ipmitool
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -115,21 +116,21 @@ func (ipmi *LanPlusIPMI) GetMode() string {
 	return "rmcp"
 }
 
-func (ipmi *LanPlusIPMI) GetCommand(args ...string) *procutils.Command {
+func (ipmi *LanPlusIPMI) GetCommand(args ...string) (*procutils.Command, context.CancelFunc) {
 	nArgs := []string{
-		"--signal=KILL",
-		fmt.Sprintf("%s", ipmi.GetDefaultTimeout()),
-		"ipmitool", "-I", "lanplus", "-H", ipmi.host,
+		"-I", "lanplus", "-H", ipmi.host,
 		"-p", fmt.Sprintf("%d", ipmi.port),
 		"-U", ipmi.user,
 		"-P", ipmi.password,
 	}
 	nArgs = append(nArgs, args...)
-	return procutils.NewCommand("timeout", nArgs...)
+	ctx, cancel := context.WithTimeout(context.Background(), ipmi.GetDefaultTimeout())
+	return procutils.NewCommandContext(ctx, "ipmitool", nArgs...), cancel
 }
 
 func (ipmi *LanPlusIPMI) ExecuteCommand(args ...string) ([]string, error) {
-	cmd := ipmi.GetCommand(args...)
+	cmd, cancel := ipmi.GetCommand(args...)
+	defer cancel()
 	log.Debugf("[LanPlusIPMI] execute command: %s", cmd.String())
 	out, err := cmd.Output()
 	if err != nil {

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -624,7 +624,7 @@ func (a StorcliAdaptor) key() string {
 }
 
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.key() != ""
+	return a.Controller >= 0 && a.name != "" && a.sn != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {
@@ -756,7 +756,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build JBOD fail: %v", err)
+	log.Errorf("Try storcli build JBOD fail: %v", err)
 	labels := []string{}
 	for _, dev := range devs {
 		labels = append(labels, GetSpecString(dev))
@@ -779,7 +779,7 @@ func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build jbod fail: %v", err)
+	log.Errorf("Try megacli build jbod fail: %v", err)
 	cmds := []string{}
 	for _, dev := range devs {
 		cmd := GetCommand("-CfgLdAdd", "-r0", fmt.Sprintf("[%s]", GetSpecString(dev)),
@@ -862,29 +862,42 @@ func (adapter *MegaRaidAdaptor) RemoveLogicVolumes() error {
 	return nil
 }
 
-/*
-def _storcli_clear_jbod_disks(self):
-    cmds = []
-    for dev in self.devs:
-        cmd = self.raid.get_command2(
-                    '/c%d/e%d/s%d' % (self.index, dev.enclosure, dev.slot),
-                    'set', 'good', 'force')
-        cmds.append(cmd)
-    logging.info('%s', cmds)
-    self.raid.term.exec_remote_commands(cmds)
-*/
+func (adapter *MegaRaidAdaptor) storcliClearJBODDisks() error {
+	errs := make([]error, 0)
+	for _, dev := range adapter.devs {
+		cmd := GetCommand2(fmt.Sprintf("/c%d/e%d/s%d", adapter.index, dev.enclosure, dev.slot), "set", "good", "force")
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "Set PD good storcli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
+}
 
 func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
 	devIds := []string{}
 	for idx, dev := range adapter.devs {
 		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage(idx)))
 	}
-	cmd := GetCommand("-PDMakeGood", fmt.Sprintf("-PhysDrv[%s]", strings.Join(devIds, ",")), "-Force", fmt.Sprintf("-a%d", adapter.index))
-	_, err := adapter.remoteRun(cmd)
-	return err
+	errs := make([]error, 0)
+	for _, devId := range devIds {
+		cmd := GetCommand("-PDMakeGood", "-PhysDrv", fmt.Sprintf("'[%s]'", devId), "-Force", fmt.Sprintf("-a%d", adapter.index))
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "PDMakeGood megacli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
 }
 
 func (adapter *MegaRaidAdaptor) clearJBODDisks() {
+	if err := adapter.megacliClearJBODDisks(); err != nil {
+		log.Errorf("megacliClearJBODDisks error: %v", err)
+		log.Infof("try storcliClearJBODDisks")
+		if err := adapter.storcliClearJBODDisks(); err != nil {
+			log.Errorf("storcliClearJBODDisks error: %v", err)
+		}
+	}
 	adapter.megacliEnableJBOD(true)
 	adapter.megacliEnableJBOD(false)
 	adapter.megacliEnableJBOD(true)

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -56,7 +56,7 @@ func (dev *Mpt2SASRaidPhyDev) parseLine(line string) bool {
 	}
 	switch key {
 	case "Drive Type":
-		if strings.HasSuffix("_HDD", val) {
+		if strings.HasSuffix(val, "_HDD") {
 			dev.Rotate = tristate.True
 		} else {
 			dev.Rotate = tristate.False

pkg/cloudcommon/consts/consts.go

@@ -35,6 +35,8 @@ var (
 	maxPagingLimit     int64 = 2048
 
 	domainizedNamespace = true
+
+	enableQuotaCheck = false
 )
 
 func SetRegion(region string) {
@@ -85,3 +87,11 @@ func SetDomainizedNamespace(domainNS bool) {
 func IsDomainizedNamespace() bool {
 	return nonDefaultDomainProjects && domainizedNamespace
 }
+
+func SetEnableQuotaCheck(val bool) {
+	enableQuotaCheck = val
+}
+
+func EnableQuotaCheck() bool {
+	return enableQuotaCheck
+}

pkg/cloudcommon/db/metadata.go

@@ -459,17 +459,17 @@ func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key s
 }
 
 func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
 		return err
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
 
-func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
+func (manager *SMetadataManager) setValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
 	// no need to lock
@@ -546,9 +546,9 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 }
 
 func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "setValues")
 	}
 
 	idStr := GetObjectIdstr(obj)
@@ -565,9 +565,9 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 	q := manager.Query().Equals("id", idStr).NotLike("key", `\_\_%`) //避免删除系统内置的metadata, _ 在mysql里面有特殊含义,需要转义
 	switch delRange {
 	case USER_TAG_PREFIX:
-		q = q.Like("key", USER_TAG_PREFIX+"%")
+		q = q.Startswith("key", USER_TAG_PREFIX)
 	case CLOUD_TAG_PREFIX:
-		q = q.Like("key", CLOUD_TAG_PREFIX+"%")
+		q = q.Startswith("key", CLOUD_TAG_PREFIX)
 	}
 	q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys)))
 	if err := FetchModelObjects(manager, q, &records); err != nil {
@@ -581,7 +581,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }

pkg/cloudcommon/db/opslog.go

@@ -117,7 +117,8 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
 	if !consts.OpsLogEnabled() {
 		return
 	}
-	if len(model.GetId()) == 0 || len(model.GetName()) == 0 {
+	if len(model.GetId()) == 0 {
+		log.Errorf("logevent for an object without ID???")
 		return
 	}
 	if action == ACT_UPDATE {

pkg/cloudcommon/db/opslog_const.go

@@ -47,8 +47,9 @@ const (
 	ACT_BACKUP_START        = "backup_start"
 	ACT_BACKUP_START_FAILED = "backup_start_fail"
 
-	ACT_FREEZE   = "freeze"
-	ACT_UNFREEZE = "unfreeze"
+	ACT_FREEZE      = "freeze"
+	ACT_FREEZE_FAIL = "freeze_fail"
+	ACT_UNFREEZE    = "unfreeze"
 
 	ACT_RESTARING    = "restarting"
 	ACT_RESTART_FAIL = "restart_fail"

pkg/cloudcommon/db/quotas/errors.go

@@ -19,9 +19,11 @@ import (
 	"strings"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SOutOfQuotaError struct {
+	scope   rbacutils.TRbacScope
 	name    string
 	limit   int
 	used    int
@@ -37,7 +39,7 @@ func (e *SOutOfQuotaError) Cause() error {
 }
 
 func (e *SOutOfQuotaError) Error() string {
-	return fmt.Sprintf("%s limit %d used %d request %d", e.name, e.limit, e.used, e.request)
+	return fmt.Sprintf("[%s.%s] limit %d used %d request %d", e.scope, e.name, e.limit, e.used, e.request)
 }
 
 func (es *SOutOfQuotaErrors) Error() string {
@@ -63,8 +65,10 @@ func NewOutOfQuotaError() *SOutOfQuotaErrors {
 	}
 }
 
-func (es *SOutOfQuotaErrors) Add(name string, limit int, used int, request int) {
+func (es *SOutOfQuotaErrors) Add(quota IQuota, name string, limit int, used int, request int) {
+	scope := quota.GetKeys().Scope()
 	e := SOutOfQuotaError{
+		scope:   scope,
 		name:    name,
 		limit:   limit,
 		used:    used,

pkg/cloudcommon/db/quotas/handler.go

@@ -132,6 +132,12 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 	ret := jsonutils.NewDict()
 
 	keys := quota.GetKeys()
+	ret.Update(jsonutils.Marshal(keys))
+	ret.Update(quota.ToJSON(""))
+
+	if !consts.EnableQuotaCheck() {
+		return ret, nil
+	}
 
 	usage := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, usage)
@@ -150,8 +156,6 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 		return nil, errors.Wrap(err, "manager.GetPendingUsages")
 	}
 
-	ret.Update(jsonutils.Marshal(keys))
-	ret.Update(quota.ToJSON(""))
 	if usage != nil {
 		ret.Update(usage.ToJSON("usage"))
 	}

pkg/cloudcommon/db/quotas/interface.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type IQuotaKeys interface {
@@ -30,6 +31,8 @@ type IQuotaKeys interface {
 	Compare(IQuotaKeys) int
 
 	OwnerId() mcclient.IIdentityProvider
+
+	Scope() rbacutils.TRbacScope
 }
 
 type IQuota interface {

pkg/cloudcommon/db/quotas/register.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -57,6 +58,10 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 }
 
 func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	if localUsage == nil {
 		return nil
 	}
@@ -65,6 +70,10 @@ func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential,
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	manager := getQuotaManager(quota)
 	err := manager.checkSetPendingQuota(ctx, userCred, quota)
 	if err != nil {
@@ -75,6 +84,9 @@ func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		cancelUsage(ctx, userCred, usage.(IQuota))
 	}
@@ -89,6 +101,9 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 }
 
 func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		addUsage(ctx, userCred, usage.(IQuota))
 	}

pkg/cloudcommon/db/quotas/usageworker.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -61,6 +62,12 @@ func isDirty(key string) bool {
 }
 
 func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan IQuota, realTime bool) {
+	if !consts.EnableQuotaCheck() {
+		go func() {
+			usageChan <- nil
+		}()
+		return
+	}
 	key := QuotaKeyString(keys)
 	setDirty(key)
 
@@ -105,6 +112,10 @@ func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan I
 }
 
 func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
+
 	log.Infof("CalculateQuotaUsages")
 	quota := manager.newQuota()
 	keys := quota.GetKeys()

pkg/cloudcommon/db/rbac.go

@@ -51,6 +51,8 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeUser:
 		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() || objOwnerId.GetUserId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && objOwnerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}
@@ -103,6 +105,8 @@ func isClassRbacAllowed(manager IModelManager, userCred mcclient.TokenCredential
 	case rbacutils.ScopeUser:
 		if ownerId != nil && ownerId.GetUserId() == objOwnerId.GetUserId() {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}

pkg/cloudcommon/db/sharablebase.go

@@ -472,6 +472,10 @@ func SharablePerformPublic(model ISharableBaseModel, ctx context.Context, userCr
 	if targetScope.HigherThan(allowScope) {
 		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", targetScope, allowScope)
 	}
+	requireScope := model.GetPublicScope()
+	if requireScope.HigherThan(allowScope) {
+		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", requireScope, allowScope)
+	}
 
 	_, err = Update(model, func() error {
 		model.SetShare(targetScope)

pkg/cloudcommon/db/taskman/localtaskworker.go

@@ -37,8 +37,8 @@ func Error2TaskData(err error) jsonutils.JSONObject {
 	return errJson
 }
 
-func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
-	localTaskWorkerMan.Run(func() {
+func LocalTaskRunWithWorkers(task ITask, proc func() (jsonutils.JSONObject, error), wm *appsrv.SWorkerManager) {
+	wm.Run(func() {
 
 		log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN STARTXXXXXXXXXXXXXXXXX")
 		defer log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN END  XXXXXXXXXXXXXXXXX")
@@ -59,3 +59,7 @@ func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
 
 	}, nil, nil)
 }
+
+func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
+	LocalTaskRunWithWorkers(task, proc, localTaskWorkerMan)
+}

pkg/cloudcommon/db/taskman/tasks.go

@@ -41,6 +41,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
@@ -815,7 +816,11 @@ func (self *STask) GetObjects() []db.IStandaloneModel {
 }
 
 func (task *STask) GetTaskRequestHeader() http.Header {
-	header := mcclient.GetTokenHeaders(task.GetUserCred())
+	userCred := task.GetUserCred()
+	if !userCred.IsValid() {
+		userCred = auth.AdminCredential()
+	}
+	header := mcclient.GetTokenHeaders(userCred)
 	header.Set(mcclient.TASK_ID, task.GetTaskId())
 	return header
 }

pkg/cloudcommon/etcd/etcd.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
@@ -45,7 +46,8 @@ type SEtcdClient struct {
 	onKeepaliveFailure func()
 	leaseLiving        bool
 
-	watchers map[string]*SEtcdWatcher
+	watchers   map[string]*SEtcdWatcher
+	watchersMu *sync.Mutex
 }
 
 func defaultOnKeepAliveFailed() {
@@ -115,6 +117,7 @@ func NewEtcdClient(opt *SEtcdOptions, onKeepaliveFailure func()) (*SEtcdClient,
 	etcdClient.leaseTtlTimeout = timeoutSeconds
 
 	etcdClient.watchers = make(map[string]*SEtcdWatcher)
+	etcdClient.watchersMu = &sync.Mutex{}
 
 	etcdClient.namespace = opt.EtcdNamspace
 
@@ -143,6 +146,10 @@ func (cli *SEtcdClient) GetClient() *clientv3.Client {
 	return cli.client
 }
 
+func (cli *SEtcdClient) SessionLiving() bool {
+	return cli.leaseLiving
+}
+
 func (cli *SEtcdClient) startSession() error {
 	ctx := context.TODO()
 
@@ -302,8 +309,10 @@ func (cli *SEtcdClient) Watch(
 	onModify TEtcdModifyEventFunc,
 	onDelete TEtcdDeleteEventFunc,
 ) error {
+	cli.watchersMu.Lock()
 	_, ok := cli.watchers[prefix]
 	if ok {
+		cli.watchersMu.Unlock()
 		return errors.Errorf("watch prefix %s already registered", prefix)
 	}
 
@@ -314,6 +323,7 @@ func (cli *SEtcdClient) Watch(
 		watcher: watcher,
 		cancel:  cancel,
 	}
+	cli.watchersMu.Unlock()
 
 	prefix = cli.getKey(prefix)
 
@@ -342,6 +352,8 @@ func (cli *SEtcdClient) Watch(
 }
 
 func (cli *SEtcdClient) Unwatch(prefix string) {
+	cli.watchersMu.Lock()
+	defer cli.watchersMu.Unlock()
 	watcher, ok := cli.watchers[prefix]
 	if ok {
 		log.Debugf("unwatch %s", prefix)

pkg/cloudcommon/options/changes.go

@@ -40,6 +40,9 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 		consts.SetDomainizedNamespace(newOpts.DomainizedNamespace)
 		changed = true
 	}
+	if oldOpts.EnableQuotaCheck != newOpts.EnableQuotaCheck {
+		consts.SetEnableQuotaCheck(newOpts.EnableQuotaCheck)
+	}
 	return changed
 }
 

pkg/cloudcommon/options/manager.go

@@ -124,10 +124,10 @@ func (manager *SOptionManager) DoSync(first bool) (time.Duration, error) {
 			appsrv.SetExitFlag()
 		}
 		copyOptions(manager.options, newOpts)
-		if first {
-			// upload config for the first time ONLY
-			manager.session.Upload()
-		}
+		// if first {
+		// upload config
+		manager.session.Upload()
+		// }
 	}
 	return manager.refreshInterval, nil
 }

pkg/cloudcommon/options/mergeconf.go

@@ -66,6 +66,8 @@ type mcclientServiceConfigSession struct {
 	session   *mcclient.ClientSession
 	serviceId string
 	config    *jsonutils.JSONDict
+
+	commonServiceId string
 }
 
 func newServiceConfigSession() IServiceConfigSession {
@@ -88,12 +90,12 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 			merged = true
 		} else {
 			// not initialized
-			s.Upload()
+			// s.Upload()
 		}
 	}
-	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
-	if len(commonServiceId) > 0 {
-		commonConf, err := getServiceConfig(s.session, commonServiceId)
+	s.commonServiceId, _ = getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
+	if len(s.commonServiceId) > 0 {
+		commonConf, err := getServiceConfig(s.session, s.commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
 		} else if commonConf != nil {
@@ -123,6 +125,11 @@ func (s *mcclientServiceConfigSession) Upload() {
 			// ignore the error
 			log.Errorf("fail to save config: %s", err)
 		}
+		_, err = modules.ServicesV3.PerformAction(s.session, s.commonServiceId, "config", nconf)
+		if err != nil {
+			// ignore the error
+			log.Errorf("fail to save common config: %s", err)
+		}
 	}
 }
 

pkg/cloudcommon/options/options.go

@@ -78,6 +78,7 @@ type BaseOptions struct {
 	IsSlaveNode        bool `help:"Slave mode"`
 	CronJobWorkerCount int  `help:"Cron job worker count" default:"4"`
 
+	EnableQuotaCheck  bool   `help:"enable quota check" default:"true"`
 	DefaultQuotaValue string `help:"default quota value" choices:"unlimit|zero|default" default:"default"`
 
 	CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"`

pkg/cloudevent/models/cloudproviders.go

@@ -262,23 +262,12 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	if err != nil {
 		return start, end, errors.Wrap(err, "self.GetProviderFactory")
 	}
-	q := CloudeventManager.Query().Equals("cloudprovider_id", self.Id).Desc("created_at")
-	count, err := q.CountWithError()
-	if err != nil {
-		return start, end, errors.Wrap(err, "q.CountWithError")
-	}
 	if !self.LastSyncTimeAt.IsZero() {
 		start = self.LastSyncTimeAt
-	} else if count == 0 {
-		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	} else {
-		event := &SCloudevent{}
-		err = q.First(event)
-		if err != nil {
-			return start, end, errors.Wrap(err, "q.First")
-		}
-		start = event.CreatedAt
+		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	}
+
 	// 避免cloudevent过长时间未运行，再次运行时记录的最后一条时间距离现在间隔太长
 	if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
 		start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)

pkg/cloudevent/service/handlers.go

@@ -55,6 +55,8 @@ func InitHandlers(app *appsrv.Application) {
 
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
+		db.Metadata,
+
 		proxy.ProxySettingManager,
 		models.CloudeventManager,
 	} {

pkg/cloudprovider/instance.go

@@ -26,6 +26,52 @@ import (
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )
 
+type SDistDefaultAccount struct {
+	// 操作系统发行版
+	OsDistribution string
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改
+	Changeable bool
+}
+
+type SOsDefaultAccount struct {
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改用户名
+	Changeable bool
+	// 禁止使用的账号
+	DisabledAccounts []string
+	// 各操作系统发行版的默认用户名信息
+	DistAccounts []SDistDefaultAccount
+}
+
+type SDefaultAccount struct {
+	Linux   SOsDefaultAccount
+	Windows SOsDefaultAccount
+}
+
+type StorageInfo struct {
+	StorageType string
+	MaxSizeGb   int
+	MinSizeGb   int
+	Resizable   bool
+	StepSizeGb  int
+}
+
+type Storage struct {
+	DataDisk []StorageInfo
+	SysDisk  []StorageInfo
+}
+
+type SInstanceCapability struct {
+	Provider   string
+	Hypervisor string
+	Storages   Storage
+
+	DefaultAccount SDefaultAccount
+}
+
 type SDiskInfo struct {
 	StorageExternalId string
 	StorageType       string

pkg/cloudprovider/resources.go

@@ -75,6 +75,7 @@ type ICloudRegion interface {
 	CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error)
 
 	CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error)
+	CreateInternetGateway() (ICloudInternetGateway, error)
 	CreateEIP(eip *SEip) (ICloudEIP, error)
 
 	GetISnapshots() ([]ICloudSnapshot, error)
@@ -437,6 +438,10 @@ type ICloudVpc interface {
 	// GetGlobalId() // 若vpc属于globalvpc,此函数返回格式必须是 'region.GetGlobalId()/vpc.GetGlobalId()'
 	ICloudResource
 
+	IsSupportSetExternalAccess() bool // 是否支持Attach互联网网关.
+	GetExternalAccessMode() string
+	AttachInternetGateway(igwId string) error
+
 	GetRegion() ICloudRegion
 	GetIsDefault() bool
 	GetCidrBlock() string
@@ -451,6 +456,10 @@ type ICloudVpc interface {
 	GetINatGateways() ([]ICloudNatGateway, error)
 }
 
+type ICloudInternetGateway interface {
+	ICloudResource
+}
+
 type ICloudWire interface {
 	ICloudResource
 	GetIVpc() ICloudVpc

pkg/cloudprovider/securitygroup.go

@@ -18,12 +18,64 @@ import (
 	"sort"
 	"strings"
 
+	"gopkg.in/fatih/set.v0"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/secrules"
+	"yunion.io/x/pkg/utils"
 )
 
-const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
-const DEFAULT_LOCAL_RULE_ID = "default_local_rule_id"
+type SecDriver interface {
+	GetDefaultSecurityGroupInRule() SecurityRule
+	GetDefaultSecurityGroupOutRule() SecurityRule
+	GetSecurityGroupRuleMaxPriority() int
+	GetSecurityGroupRuleMinPriority() int
+	IsOnlySupportAllowRules() bool
+}
+
+func NewSecRuleInfo(driver SecDriver) SecRuleInfo {
+	return SecRuleInfo{
+		InDefaultRule:           driver.GetDefaultSecurityGroupInRule(),
+		OutDefaultRule:          driver.GetDefaultSecurityGroupOutRule(),
+		MinPriority:             driver.GetSecurityGroupRuleMinPriority(),
+		MaxPriority:             driver.GetSecurityGroupRuleMaxPriority(),
+		IsOnlySupportAllowRules: driver.IsOnlySupportAllowRules(),
+	}
+}
+
+const DEFAULT_DEST_RULE_ID = "default_dest_rule_id"
+const DEFAULT_SRC_RULE_ID = "default_src_rule_id"
+
+type SecRuleInfo struct {
+	InDefaultRule           SecurityRule
+	OutDefaultRule          SecurityRule
+	Rules                   SecurityRuleSet
+	MinPriority             int
+	MaxPriority             int
+	IsOnlySupportAllowRules bool
+}
+
+func (r SecRuleInfo) AddDefaultRule(d SecRuleInfo, inRules, outRules []SecurityRule, isSrc bool) ([]SecurityRule, []SecurityRule) {
+	min, max := r.MinPriority, r.MaxPriority
+	r.InDefaultRule.Priority = min + 1
+	r.OutDefaultRule.Priority = min + 1
+	if max >= min {
+		r.InDefaultRule.Priority = min - 1
+		r.OutDefaultRule.Priority = min - 1
+	}
+
+	if isSrc {
+		r.InDefaultRule.Id = DEFAULT_SRC_RULE_ID
+		r.OutDefaultRule.Id = DEFAULT_SRC_RULE_ID
+	} else {
+		r.InDefaultRule.ExternalId = DEFAULT_DEST_RULE_ID
+		r.OutDefaultRule.ExternalId = DEFAULT_DEST_RULE_ID
+	}
+
+	inRules = append(inRules, r.InDefaultRule)
+	outRules = append(outRules, r.OutDefaultRule)
+	return inRules, outRules
+}
 
 type SecurityGroupFilterOptions struct {
 	VpcId     string
@@ -43,57 +95,26 @@ type SecurityRule struct {
 	secrules.SecurityRule
 	Name       string
 	ExternalId string
+	Id         string
 }
 
-type LocalSecurityRule struct {
-	secrules.SecurityRule
-	ExternalId string
-}
-
-func (r LocalSecurityRule) String() string {
-	return r.SecurityRule.String()
-}
-
-type LocalSecurityRuleSet []LocalSecurityRule
-
-func (srs LocalSecurityRuleSet) Len() int {
-	return len(srs)
-}
-
-func (srs LocalSecurityRuleSet) Swap(i, j int) {
-	srs[i], srs[j] = srs[j], srs[i]
-}
-
-func (srs LocalSecurityRuleSet) Less(i, j int) bool {
-	if srs[i].Priority > srs[j].Priority {
-		return true
-	} else if srs[i].Priority == srs[j].Priority {
-		return srs[i].String() < srs[j].String()
-	}
-	return false
-}
-
-func (srs LocalSecurityRuleSet) AllowList() secrules.SecurityRuleSet {
-	rules := secrules.SecurityRuleSet{}
-	for _, r := range srs {
-		rules = append(rules, r.SecurityRule)
-	}
-	return rules.AllowList()
-}
-
-type TPriorityOrder int
-
-var (
-	PriorityOrderByDesc = TPriorityOrder(1)
-	PriorityOrderByAsc  = TPriorityOrder(-1)
-)
-
 func (r SecurityRule) String() string {
 	return r.SecurityRule.String()
 }
 
 type SecurityRuleSet []SecurityRule
 
+func (rules SecurityRuleSet) Split() (in, out SecurityRuleSet) {
+	for i := 0; i < len(rules); i++ {
+		if rules[i].Direction == secrules.DIR_IN {
+			in = append(in, rules[i])
+		} else {
+			out = append(out, rules[i])
+		}
+	}
+	return
+}
+
 func (srs SecurityRuleSet) Len() int {
 	return len(srs)
 }
@@ -114,234 +135,191 @@ func (srs SecurityRuleSet) AllowList() secrules.SecurityRuleSet {
 	return rules.AllowList()
 }
 
-func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr string, order TPriorityOrder, min, max int, onlyAllowRules bool) []SecurityRule {
-	if defaultRule.String() == localRuleStr {
-		return rules
+func (srs SecurityRuleSet) Debug() {
+	for i := 0; i < len(srs); i++ {
+		log.Debugf("Name: %s id: %s external_id: %s priority: %d %s", srs[i].Name, srs[i].Id, srs[i].ExternalId, srs[i].Priority, srs[i].String())
 	}
-	defaultRule.ExternalId = DEFAULT_CLOUD_RULE_ID
-	if order == PriorityOrderByDesc {
-		defaultRule.Priority = min
-	} else {
-		defaultRule.Priority = max
-	}
-	defaultRule.Priority -= int(order)
-	return append(rules, defaultRule)
 }
 
-func SortSecurityRule(rules SecurityRuleSet, order TPriorityOrder, onlyAllowRules bool) {
-	if onlyAllowRules {
-		sort.Sort(rules)
-		return
-	}
-	if order == PriorityOrderByAsc {
+func SortSecurityRule(rules SecurityRuleSet, max, min int, isAsc, onlyAllowRules bool) {
+	if (max >= min || onlyAllowRules) && !isAsc {
 		sort.Sort(sort.Reverse(rules))
 		return
 	}
 	sort.Sort(rules)
+	return
 }
 
-func CompareRules(
-	minPriority, maxPriority int, order TPriorityOrder,
-	localRules secrules.SecurityRuleSet, remoteRules []SecurityRule,
-	defaultInRule, defaultOutRule SecurityRule,
-	onlyAllowRules bool, debug bool,
-) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
-	localInRules := LocalSecurityRuleSet{}
-	localOutRules := LocalSecurityRuleSet{}
-	for i := range localRules {
-		localRule := LocalSecurityRule{}
-		localRule.SecurityRule = localRules[i]
-		if localRules[i].Direction == secrules.DIR_IN {
-			localInRules = append(localInRules, localRule)
-		} else {
-			localOutRules = append(localOutRules, localRule)
-		}
+func isAllowListEqual(src, dest secrules.SecurityRuleSet) bool {
+	if len(src) != len(dest) {
+		return false
 	}
-	inRules := SecurityRuleSet{}
-	outRules := SecurityRuleSet{}
-	for i := 0; i < len(remoteRules); i++ {
-		if remoteRules[i].Direction == secrules.DIR_IN {
-			inRules = append(inRules, remoteRules[i])
-		} else {
-			outRules = append(outRules, remoteRules[i])
-		}
+	s1, s2 := set.New(set.ThreadSafe), set.New(set.ThreadSafe)
+	for i := 0; i < len(src); i++ {
+		s1.Add(src[i].String())
+		s2.Add(dest[i].String())
 	}
-	var inCommon, outCommon = inRules, outRules
+	return s1.IsEqual(s2)
+}
 
-	defaultLocalInRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
-	defaultLocalInRule.SecurityRule = *secrules.MustParseSecurityRule("in:deny any")
-	defaultLocalOutRule := LocalSecurityRule{ExternalId: DEFAULT_LOCAL_RULE_ID}
-	defaultLocalOutRule.SecurityRule = *secrules.MustParseSecurityRule("out:allow any")
+func CompareRules(src, dest SecRuleInfo, debug bool) (common, inAdds, outAdds, inDels, outDels SecurityRuleSet) {
+	srcInRules, srcOutRules := src.Rules.Split()
+	destInRules, destOutRules := dest.Rules.Split()
 
-	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
-	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+	srcInRules, srcOutRules = src.AddDefaultRule(dest, srcInRules, srcOutRules, true)
+	destInRules, destOutRules = dest.AddDefaultRule(src, destInRules, destOutRules, false)
 
-	defaultInEquals, defaultOutEquals := true, true
-	if defaultLocalInRule.String() != defaultInRule.String() {
-		localInRules = append(localInRules, defaultLocalInRule)
-		defaultInEquals = false
-	}
-	if defaultLocalOutRule.String() != defaultOutRule.String() {
-		localOutRules = append(localOutRules, defaultLocalOutRule)
-		defaultOutEquals = false
+	if debug {
+		log.Debugf("src in rules: ")
+		srcInRules.Debug()
 	}
 
-	sort.Sort(localInRules)
-	sort.Sort(localOutRules)
+	// AllowList 需要优先级从高到低排序
+	SortSecurityRule(srcInRules, src.MaxPriority, src.MinPriority, false, src.IsOnlySupportAllowRules)
+	SortSecurityRule(srcOutRules, src.MaxPriority, src.MinPriority, false, src.IsOnlySupportAllowRules)
 
-	localInAllowList := localInRules.AllowList()
-	localOutAllowList := localOutRules.AllowList()
-	_localInRules := LocalSecurityRuleSet{}
-	for i := range localInAllowList {
-		rule := LocalSecurityRule{}
-		rule.SecurityRule = localInAllowList[i]
-		_localInRules = append(_localInRules, rule)
-	}
-	_localOutRules := LocalSecurityRuleSet{}
-	for i := range localOutAllowList {
-		rule := LocalSecurityRule{}
-		rule.SecurityRule = localOutAllowList[i]
-		_localOutRules = append(_localOutRules, rule)
-	}
-	if onlyAllowRules {
-		localOutRules, localInRules = _localOutRules, _localInRules
-	}
-	if len(_localInRules) < len(localInRules) {
-		localInRules = _localInRules
-	}
-	if len(_localOutRules) < len(localOutRules) {
-		localOutRules = _localOutRules
-	}
+	SortSecurityRule(destInRules, dest.MaxPriority, dest.MinPriority, false, dest.IsOnlySupportAllowRules)
+	SortSecurityRule(destOutRules, dest.MaxPriority, dest.MinPriority, false, dest.IsOnlySupportAllowRules)
 
-	SortSecurityRule(inRules, order, onlyAllowRules)
-	SortSecurityRule(outRules, order, onlyAllowRules)
+	srcInAllowList := srcInRules.AllowList()
+	srcOutAllowList := srcOutRules.AllowList()
+
+	destInAllowList := destInRules.AllowList()
+	destOutAllowList := destOutRules.AllowList()
+	inEquals, outEquals := isAllowListEqual(srcInAllowList, destInAllowList), isAllowListEqual(srcOutAllowList, destOutAllowList)
 
-	inAllowList := inRules.AllowList()
-	outAllowList := outRules.AllowList()
-	inEquals, outEquals := inAllowList.Equals(localInAllowList), outAllowList.Equals(localOutAllowList)
 	if inEquals && outEquals {
 		return
 	}
 
-	// priority从小到大排列(从默认规则开始对比)
-	sort.Sort(sort.Reverse(localInRules))
-	sort.Sort(sort.Reverse(localOutRules))
-
-	sort.Sort(sort.Reverse(inRules))
-	sort.Sort(sort.Reverse(outRules))
-
-	startPriority := minPriority - 1
-	if order == PriorityOrderByAsc {
-		startPriority = maxPriority + 1
+	if debug {
+		log.Debugf("In: src: %s dest: %s result: %v", srcInAllowList.String(), destInAllowList.String(), inEquals)
+		log.Debugf("Out: src: %s dest: %s result: %v", srcOutAllowList.String(), destOutAllowList.String(), outEquals)
 	}
 
-	var addPriority = func(priority int, order TPriorityOrder, inc int, min, max int, onlyAllowRules bool) int {
+	var tryUseAllowList = func(defaultRule SecurityRule, allowList secrules.SecurityRuleSet, rules SecurityRuleSet, isOnlyAllowList bool) SecurityRuleSet {
+		if len(allowList) < len(rules) || isOnlyAllowList {
+			rules = SecurityRuleSet{}
+			for i := range allowList {
+				rule := SecurityRule{}
+				rule.SecurityRule = allowList[i]
+				rules = append(rules, rule)
+			}
+
+			if !utils.IsInStringArray(allowList.String(), []string{
+				"",
+				"in:allow any",
+				"out:allow any",
+				"in:deny any",
+				"out:deny any",
+			}) && strings.HasSuffix(defaultRule.SecurityRule.String(), "deny any") {
+				rules = append(rules, defaultRule)
+			}
+		}
+		return rules
+	}
+
+	srcInRules = tryUseAllowList(src.InDefaultRule, srcInAllowList, srcInRules, dest.IsOnlySupportAllowRules)
+	srcOutRules = tryUseAllowList(src.OutDefaultRule, srcOutAllowList, srcOutRules, dest.IsOnlySupportAllowRules)
+
+	if inEquals {
+		srcInRules, destInRules = []SecurityRule{}, []SecurityRule{}
+	}
+	if outEquals {
+		srcOutRules, destOutRules = []SecurityRule{}, []SecurityRule{}
+	}
+
+	if debug {
+		log.Debugf("src in rules: ")
+		srcInRules.Debug()
+	}
+
+	// 默认从优先级低到高比较
+	SortSecurityRule(srcInRules, src.MaxPriority, src.MinPriority, true, src.IsOnlySupportAllowRules)
+	SortSecurityRule(srcOutRules, src.MaxPriority, src.MinPriority, true, src.IsOnlySupportAllowRules)
+
+	SortSecurityRule(destInRules, dest.MaxPriority, dest.MinPriority, true, dest.IsOnlySupportAllowRules)
+	SortSecurityRule(destOutRules, dest.MaxPriority, dest.MinPriority, true, dest.IsOnlySupportAllowRules)
+
+	var addPriority = func(priority int, min, max int, onlyAllowRules bool) int {
 		if onlyAllowRules {
-			return 0
+			return priority
 		}
-		inc = inc * int(order) //+ int(order)
-		priority += inc
-		if priority < min {
-			return min
+		inc := 1
+		if max < min {
+			max, min, inc = min, max, -1
 		}
-		if priority > max {
-			return max
+		if priority >= max || priority <= min {
+			return priority
 		}
-		return priority
+		return priority + inc
 	}
 
-	var getInitPriority = func(init, min, max int) int {
-		if init < min || init > max {
-			return (min + max) / 2
-		}
-		return init
-	}
-
-	var compare = func(localRules LocalSecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
-		i, j, inc, prePriority := 0, 0, 1, 0
-		for i < len(localRules) || j < len(remoteRules) {
-			if i < len(localRules) && j < len(remoteRules) {
-				ruleStr := remoteRules[j].String()
-				localRuleStr := localRules[i].String()
+	var _compare = func(srcRules SecurityRuleSet, destRules SecurityRuleSet) (common, add, del SecurityRuleSet) {
+		i, j, priority := 0, 0, (dest.MinPriority-1+dest.MaxPriority)/2
+		for i < len(srcRules) || j < len(destRules) {
+			if i < len(srcRules) && j < len(destRules) {
+				destRuleStr := destRules[j].String()
+				srcRuleStr := srcRules[i].String()
 				if debug {
-					log.Debugf("compare local priority(%d) %s -> remote name(%s) priority(%d) %s\n", localRules[i].Priority, localRules[i].String(), remoteRules[j].Name, remoteRules[j].Priority, remoteRules[j].String())
+					log.Debugf("compare src %s(%s) priority(%d) %s -> dest name(%s) %s(%s) priority(%d) %s\n",
+						srcRules[i].Id, srcRules[i].ExternalId, srcRules[i].Priority, srcRules[i].String(),
+						destRules[j].Name, destRules[j].ExternalId, destRules[j].Id, destRules[j].Priority, destRules[j].String())
 				}
-				cmp := strings.Compare(ruleStr, localRuleStr)
+				cmp := strings.Compare(destRuleStr, srcRuleStr)
 				if cmp == 0 {
-					prePriority = remoteRules[j].Priority
-					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
-						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
-					}
-					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-						common = append(common, remoteRules[j])
+					destRules[j].Id = srcRules[i].Id
+					common = append(common, destRules[j])
+					if destRules[j].ExternalId != DEFAULT_DEST_RULE_ID {
+						priority = destRules[j].Priority
 					}
 					i++
 					j++
 				} else if cmp < 0 {
-					if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
-						del = append(del, remoteRules[j])
-					}
+					del = append(del, destRules[j])
 					j++
 				} else {
-					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
-					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-					if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-						(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-						(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-						add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
-					}
+					priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+					srcRules[i].Priority = priority
+					add = append(add, srcRules[i])
 					i++
-					inc++
-				}
-			} else if i >= len(localRules) {
-				if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
-					del = append(del, remoteRules[j])
 				}
+			} else if i >= len(srcRules) {
+				del = append(del, destRules[j])
 				j++
-			} else if j >= len(remoteRules) {
-				initPriority := startPriority
-				if len(remoteRules) > 0 {
-					initPriority = remoteRules[len(remoteRules)-1].Priority
-				}
-				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
-				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
-				if localRules[i].ExternalId != DEFAULT_LOCAL_RULE_ID ||
-					(localRules[i].Direction == secrules.DIR_IN && !defaultInEquals) ||
-					(localRules[i].Direction == secrules.DIR_OUT && !defaultOutEquals) {
-					add = append(add, SecurityRule{SecurityRule: localRules[i].SecurityRule})
-				}
+			} else if j >= len(destRules) {
+				priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+				srcRules[i].Priority = priority
+				add = append(add, srcRules[i])
 				i++
-				inc++
 			}
 		}
 		return
 	}
 
 	type rulePair struct {
-		localRules  LocalSecurityRuleSet
-		remoteRules []SecurityRule
-		protocol    string
+		srcRules  SecurityRuleSet
+		destRules SecurityRuleSet
+		protocol  string
 	}
 
-	var splitRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) []rulePair {
+	var splitRules = func(src, dest SecurityRuleSet) []rulePair {
 		rules := map[string]rulePair{}
-		for _, r := range localRules {
+		for _, r := range src {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{srcRules: SecurityRuleSet{}, destRules: SecurityRuleSet{}, protocol: r.Protocol}
 			}
-			pair.localRules = append(pair.localRules, r)
+			pair.srcRules = append(pair.srcRules, r)
 			rules[r.Protocol] = pair
 		}
 
-		for _, r := range remoteRules {
+		for _, r := range dest {
 			pair, ok := rules[r.Protocol]
 			if !ok {
-				pair = rulePair{localRules: LocalSecurityRuleSet{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+				pair = rulePair{srcRules: SecurityRuleSet{}, destRules: SecurityRuleSet{}, protocol: r.Protocol}
 			}
-			pair.remoteRules = append(pair.remoteRules, r)
+			pair.destRules = append(pair.destRules, r)
 			rules[r.Protocol] = pair
 		}
 
@@ -352,23 +330,61 @@ func CompareRules(
 		return ret
 	}
 
-	var compareRules = func(localRules LocalSecurityRuleSet, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
-		pairs := splitRules(localRules, remoteRules)
+	var compare = func(src, dest SecurityRuleSet) (common, added, dels SecurityRuleSet) {
+		pairs := splitRules(src, dest)
 		for _, r := range pairs {
-			_common, _add, _dels := compare(r.localRules, r.remoteRules)
+			_common, _add, _dels := _compare(r.srcRules, r.destRules)
 			common = append(common, _common...)
-			add = append(add, _add...)
+			added = append(added, _add...)
 			dels = append(dels, _dels...)
 		}
 		return
 	}
 
-	if !inEquals {
-		inCommon, inAdds, inDels = compareRules(localInRules, inRules)
+	var inCommon, outCommon SecurityRuleSet
+	inCommon, inAdds, inDels = compare(srcInRules, destInRules)
+	outCommon, outAdds, outDels = compare(srcOutRules, destOutRules)
+
+	var handleDefaultRules = func(removed, added []SecurityRule, isOnlyAllowList bool) ([]SecurityRule, []SecurityRule) {
+		ret := []SecurityRule{}
+		for _, rule := range removed {
+			if rule.ExternalId == DEFAULT_DEST_RULE_ID {
+				if debug {
+					log.Debugf("remove dest default rule: %s external id %s priority: %d", rule.String(), rule.ExternalId, rule.Priority)
+				}
+				if rule.Action == secrules.SecurityRuleDeny && isOnlyAllowList {
+					continue
+				}
+				switch rule.Action {
+				case secrules.SecurityRuleDeny:
+					rule.Action = secrules.SecurityRuleAllow
+				case secrules.SecurityRuleAllow:
+					rule.Action = secrules.SecurityRuleDeny
+				}
+				rule.Priority = dest.MinPriority
+
+				find := false
+				for i := range added {
+					if added[i].String() == rule.String() {
+						find = true
+						break
+					}
+				}
+				if !find {
+					if debug {
+						log.Debugf("add new default rule: %s external id %s priority: %d", rule.String(), rule.ExternalId, rule.Priority)
+					}
+					added = append(added, rule)
+				}
+			} else {
+				ret = append(ret, rule)
+			}
+		}
+		return ret, added
 	}
-	if !outEquals {
-		outCommon, outAdds, outDels = compareRules(localOutRules, outRules)
-	}
-	common = append(inCommon, outCommon...)
+
+	inDels, inAdds = handleDefaultRules(inDels, inAdds, dest.IsOnlySupportAllowRules)
+	outDels, outAdds = handleDefaultRules(outDels, outAdds, dest.IsOnlySupportAllowRules)
+	common, _ = handleDefaultRules(append(inCommon, outCommon...), []SecurityRule{}, dest.IsOnlySupportAllowRules)
 	return
 }

pkg/compute/guestdrivers/aliyun.go

@@ -164,6 +164,39 @@ func (self *SAliyunGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_READY
 }
 
+func (self *SAliyunGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_EFFICIENCY, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD_PL2, MaxSizeGb: 32768, MinSizeGb: 461, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD_PL3, MaxSizeGb: 32768, MinSizeGb: 1261, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PUBLIC_CLOUD, MaxSizeGb: 2000, MinSizeGb: 5, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_EPHEMERAL_SSD, MaxSizeGb: 800, MinSizeGb: 5, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_EFFICIENCY, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_ESSD, MaxSizeGb: 500, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
+			},
+		},
+	}
+}
+
 func (self *SAliyunGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/aws.go

@@ -86,6 +86,39 @@ func (self *SAwsGuestDriver) GetWindowsUserDataType() string {
 	return cloudprovider.CLOUD_EC2
 }
 
+func (self *SAwsGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SAwsGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	// return fetchAwsUserName(desc)
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/azure.go

@@ -222,6 +222,35 @@ func (self *SAzureGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_READY
 }
 
+func (self *SAzureGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_AZURE_DEFAULT_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARDSSD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PREMIUM_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARDSSD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_PREMIUM_LRS, MaxSizeGb: 32767, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SAzureGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	return api.VM_AZURE_DEFAULT_LOGIN_USER
 }

pkg/compute/guestdrivers/baremetals.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
@@ -56,6 +57,13 @@ func (self *SBaremetalGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_ONECLOUD
 }
 
+func (self *SBaremetalGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+	}
+}
+
 func (self *SBaremetalGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/base.go

@@ -398,3 +398,7 @@ func (self *SBaseGuestDriver) RequestRemoteUpdate(ctx context.Context, guest *mo
 	// nil ops
 	return nil
 }
+
+func (self *SBaseGuestDriver) ValidateDetachNetwork(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
+	return nil
+}

pkg/compute/guestdrivers/container.go

@@ -24,6 +24,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -57,6 +58,13 @@ func (self *SContainerDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_ONECLOUD
 }
 
+func (self *SContainerDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+	}
+}
+
 // for backward compatibility, deprecated driver
 func (self *SContainerDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}

pkg/compute/guestdrivers/ctyun.go

@@ -39,6 +39,23 @@ func (self *SCtyunGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_CTYUN
 }
 
+func (self *SCtyunGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SCtyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/esxi.go

@@ -65,6 +65,23 @@ func (self *SESXiGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_VMWARE
 }
 
+func (self *SESXiGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     true,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SESXiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/google.go

@@ -65,6 +65,33 @@ func (self *SGoogleGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_GOOGLE
 }
 
+func (self *SGoogleGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_SSD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_STANDARD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_BALANCED, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_SSD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_STANDARD, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_GOOGLE_PD_BALANCED, MaxSizeGb: 65536, MinSizeGb: 10, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SGoogleGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_GOOGLE_PD_STANDARD
 }

pkg/compute/guestdrivers/huawei.go

@@ -110,6 +110,35 @@ func (self *SHuaweiGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_RUNNING
 }
 
+func (self *SHuaweiGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SHuaweiGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/kvm.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -68,6 +69,22 @@ func (self *SKVMGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, own
 	return keys
 }
 
+func (self *SKVMGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     true,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+	}
+}
+
 func (self *SKVMGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_LOCAL
 }
@@ -629,3 +646,10 @@ func (self *SKVMGuestDriver) CheckLiveMigrate(guest *models.SGuest, userCred mcc
 	}
 	return nil
 }
+
+func (self *SKVMGuestDriver) ValidateDetachNetwork(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
+	if guest.Status == api.VM_RUNNING && guest.GetMetadata("hot_remove_nic", nil) != "enable" {
+		return httperrors.NewBadRequestError("Guest %s can't hot remove nic", guest.GetName())
+	}
+	return nil
+}

pkg/compute/guestdrivers/openstack.go

@@ -64,6 +64,23 @@ func (self *SOpenStackGuestDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_OPENSTACK
 }
 
+func (self *SOpenStackGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+	}
+}
+
 func (self *SOpenStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
 	keys := models.SComputeResourceKeys{}
 	keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)

pkg/compute/guestdrivers/qcloud.go

@@ -243,6 +243,43 @@ func (self *SQcloudGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }
 
+func (self *SQcloudGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+				Changeable:     false,
+				DistAccounts: []cloudprovider.SDistDefaultAccount{
+					{
+						OsDistribution: "Ubuntu",
+						DefaultAccount: "ubuntu",
+						Changeable:     false,
+					},
+				},
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+				Changeable:     false,
+			},
+		},
+		Storages: cloudprovider.Storage{
+			DataDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_BASIC, MaxSizeGb: 16000, MinSizeGb: 10, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_PREMIUM, MaxSizeGb: 16000, MinSizeGb: 50, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 16000, MinSizeGb: 100, StepSizeGb: 10, Resizable: true},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_HSSD, MaxSizeGb: 32000, MinSizeGb: 20, StepSizeGb: 10, Resizable: true},
+			},
+			SysDisk: []cloudprovider.StorageInfo{
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_BASIC, MaxSizeGb: 500, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_PREMIUM, MaxSizeGb: 1024, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+				cloudprovider.StorageInfo{StorageType: api.STORAGE_CLOUD_SSD, MaxSizeGb: 500, MinSizeGb: 50, StepSizeGb: 10, Resizable: false},
+			},
+		},
+	}
+}
+
 func (self *SQcloudGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.ImageType == "system" {

pkg/compute/guestdrivers/ucloud.go

@@ -96,6 +96,21 @@ func (self *SUCloudGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	return nil
 }
 
+func (self *SUCloudGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+	}
+}
+
 func (self *SUCloudGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	if desc.OsType == "Windows" {
 		return "Administrator"

pkg/compute/guestdrivers/zstack.go

@@ -158,6 +158,21 @@ func (self *SZStackGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
 	return true
 }
 
+func (self *SZStackGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
+	return cloudprovider.SInstanceCapability{
+		Hypervisor: self.GetHypervisor(),
+		Provider:   self.GetProvider(),
+		DefaultAccount: cloudprovider.SDefaultAccount{
+			Linux: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
+			},
+			Windows: cloudprovider.SOsDefaultAccount{
+				DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
+			},
+		},
+	}
+}
+
 func (self *SZStackGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/hostdrivers/base.go

@@ -153,6 +153,7 @@ func (self *SBaseHostDriver) FinishConvert(userCred mcclient.TokenCredential, ho
 	bs := host.GetBaremetalstorage().GetStorage()
 	bs.SetStatus(userCred, api.STORAGE_OFFLINE, "")
 	db.Update(host, func() error {
+		host.Name = guest.GetName()
 		host.CpuReserved = 0
 		host.MemReserved = 0
 		host.AccessIp = guest.GetRealIPs()[0]

pkg/compute/hostdrivers/kvm.go

@@ -155,18 +155,18 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod
 	params := task.GetParams()
 	imageId, err := params.GetString("image_id")
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Get image_id from params")
 	}
 	format, _ := params.GetString("format")
 	isForce := jsonutils.QueryBoolean(params, "is_force", false)
-	obj, err := models.CachedimageManager.FetchById(imageId)
-	if err != nil {
-		return err
-	}
-	cacheImage := obj.(*models.SCachedimage)
-	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_HYPERVISOR, []string{host.Id}, []interface{}{host.GetZone()})
-	if err != nil {
-		return err
+
+	srcHostId, _ := params.GetString("source_host_id")
+	var srcHost *models.SHost
+	if srcHostId != "" {
+		srcHost = models.HostManager.FetchHostById(srcHostId)
+		if srcHost == nil {
+			return errors.Errorf("Source host %s not found", srcHostId)
+		}
 	}
 
 	type contentStruct struct {
@@ -181,17 +181,33 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod
 	content.ImageId = imageId
 	content.Format = format
 
+	obj, err := models.CachedimageManager.FetchById(imageId)
+	if err != nil {
+		return errors.Wrapf(err, "Fetch cached image by image_id %s", imageId)
+	}
+	cacheImage := obj.(*models.SCachedimage)
+	rangeObjs := []interface{}{host.GetZone()}
+	if srcHost != nil {
+		rangeObjs = append(rangeObjs, srcHost)
+	}
+	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_HYPERVISOR, []string{host.Id}, rangeObjs)
+	if err != nil {
+		return errors.Wrapf(err, "Choose source storagecache")
+	}
 	if srcHostCacheImage != nil {
 		err = srcHostCacheImage.AddDownloadRefcount()
 		if err != nil {
 			return err
 		}
+
 		srcHost, err := srcHostCacheImage.GetHost()
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "Get storage cached image %s host", srcHostCacheImage.GetId())
 		}
 		content.SrcUrl = fmt.Sprintf("%s/download/images/%s", srcHost.ManagerUri, imageId)
+
 	}
+
 	url := fmt.Sprintf("%s/disks/image_cache", host.ManagerUri)
 
 	if isForce {
@@ -205,7 +221,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod
 
 	_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "POST %s", url)
 	}
 	return nil
 }

pkg/compute/models/cachedimages.go

@@ -166,6 +166,11 @@ func (self *SCachedimage) GetHypervisor() string {
 	return osType
 }
 
+func (self *SCachedimage) GetChecksum() string {
+	checksum, _ := self.Info.GetString("checksum")
+	return checksum
+}
+
 func (self *SCachedimage) getStoragecacheQuery() *sqlchemy.SQuery {
 	q := StoragecachedimageManager.Query().Equals("cachedimage_id", self.Id)
 	return q
@@ -442,6 +447,8 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 
 	for _, rangeObj := range rangeObjs {
 		switch v := rangeObj.(type) {
+		case *SHost:
+			q = q.Filter(sqlchemy.Equals(host.Field("id"), v.Id))
 		case *SZone:
 			q = q.Filter(sqlchemy.Equals(host.Field("zone_id"), v.Id))
 		case *SCloudprovider:

pkg/compute/models/capabilities.go

@@ -72,6 +72,8 @@ type SCapabilities struct {
 	StorageTypes3     map[string]map[string]*SimpleStorageInfo `json:",allowempty"`
 	DataStorageTypes2 map[string][]string                      `json:",allowempty"`
 	DataStorageTypes3 map[string]map[string]*SimpleStorageInfo `json:",allowempty"`
+
+	InstanceCapabilities []cloudprovider.SInstanceCapability
 }
 
 func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, region *SCloudregion, zone *SZone) (SCapabilities, error) {
@@ -104,6 +106,13 @@ func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, que
 		domainId = ""
 	}
 	capa.Hypervisors = getHypervisors(region, zone, domainId)
+	capa.InstanceCapabilities = []cloudprovider.SInstanceCapability{}
+	for _, hypervisor := range capa.Hypervisors {
+		driver := GetDriver(hypervisor)
+		if driver != nil {
+			capa.InstanceCapabilities = append(capa.InstanceCapabilities, driver.GetInstanceCapability())
+		}
+	}
 	getBrands(region, zone, domainId, &capa)
 	// capa.Brands, capa.ComputeEngineBrands, capa.NetworkManageBrands, capa.ObjectStorageBrands = a, c, n, o
 	capa.ResourceTypes = getResourceTypes(region, zone, domainId)

pkg/compute/models/cloudaccounts.go

@@ -2532,7 +2532,7 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 	cloudaccountPendingSyncs[account.Id] = struct{}{}
 
 	RunSyncCloudAccountTask(func() {
-		func() {
+		defer func() {
 			cloudaccountPendingSyncsMutex.Lock()
 			defer cloudaccountPendingSyncsMutex.Unlock()
 			delete(cloudaccountPendingSyncs, account.Id)
@@ -3254,7 +3254,7 @@ func (self *SCloudaccount) SyncAccountResources(ctx context.Context, userCred mc
 		return errors.Wrapf(err, "GetProvider")
 	}
 	if cloudprovider.IsSupportProject(provider) {
-		return func() error {
+		err = func() error {
 			lockman.LockRawObject(ctx, "projects", self.Id)
 			defer lockman.ReleaseRawObject(ctx, "projects", self.Id)
 
@@ -3266,10 +3266,13 @@ func (self *SCloudaccount) SyncAccountResources(ctx context.Context, userCred mc
 			log.Infof("Sync project for cloudaccount %s result: %s", self.Name, result.Result())
 			return nil
 		}()
+		if err != nil {
+			log.Errorf("sync project for account %s error: %v", self.Name, err)
+		}
 	}
 
 	if cloudprovider.IsSupportDnsZone(provider) {
-		return func() error {
+		err = func() error {
 			lockman.LockRawObject(ctx, "dns_zones", self.Id)
 			defer lockman.ReleaseRawObject(ctx, "dns_zones", self.Id)
 
@@ -3294,6 +3297,9 @@ func (self *SCloudaccount) SyncAccountResources(ctx context.Context, userCred mc
 			}
 			return nil
 		}()
+		if err != nil {
+			log.Errorf("sync dns zone for account %s error: %v", self.Name, err)
+		}
 	}
 
 	return nil

pkg/compute/models/cloudproviders.go

@@ -1489,27 +1489,53 @@ func (self *SCloudprovider) PerformDisable(ctx context.Context, userCred mcclien
 	return nil, nil
 }
 
+func (manager *SCloudproviderManager) filterByDomainId(q *sqlchemy.SQuery, domainId string) *sqlchemy.SQuery {
+	subq := db.SharedResourceManager.Query("resource_id")
+	subq = subq.Equals("resource_type", CloudaccountManager.Keyword())
+	subq = subq.Equals("target_project_id", domainId)
+	subq = subq.Equals("target_type", db.SharedTargetDomain)
+
+	cloudaccounts := CloudaccountManager.Query().SubQuery()
+	q = q.Join(cloudaccounts, sqlchemy.Equals(
+		q.Field("cloudaccount_id"),
+		cloudaccounts.Field("id"),
+	))
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.AND(
+			sqlchemy.Equals(q.Field("domain_id"), domainId),
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
+		),
+		sqlchemy.AND(
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
+			sqlchemy.OR(
+				sqlchemy.AND(
+					sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeNone),
+					sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+				),
+				sqlchemy.AND(
+					sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeDomain),
+					sqlchemy.OR(
+						sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+						sqlchemy.In(cloudaccounts.Field("id"), subq.SubQuery()),
+					),
+				),
+				sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeSystem),
+			),
+		),
+		sqlchemy.AND(
+			sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
+		),
+	))
+	return q
+}
+
 func (manager *SCloudproviderManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
 	if owner != nil {
 		switch scope {
 		case rbacutils.ScopeProject, rbacutils.ScopeDomain:
 			if len(owner.GetProjectDomainId()) > 0 {
-				cloudaccounts := CloudaccountManager.Query().SubQuery()
-				q = q.Join(cloudaccounts, sqlchemy.Equals(
-					q.Field("cloudaccount_id"),
-					cloudaccounts.Field("id"),
-				))
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.AND(
-						sqlchemy.Equals(q.Field("domain_id"), owner.GetProjectDomainId()),
-						sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
-					),
-					sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
-					sqlchemy.AND(
-						sqlchemy.Equals(cloudaccounts.Field("domain_id"), owner.GetProjectDomainId()),
-						sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
-					),
-				))
+				q = manager.filterByDomainId(q, owner.GetProjectDomainId())
 			}
 		}
 	}

pkg/compute/models/cloudregions.go

@@ -610,28 +610,17 @@ func (manager *SCloudregionManager) InitializeData() error {
 }
 
 func getCloudRegionIdByDomainId(domainId string) *sqlchemy.SSubQuery {
-	accounts := CloudaccountManager.Query().SubQuery()
 	cloudproviderregions := CloudproviderRegionManager.Query().SubQuery()
-	providers := CloudproviderManager.Query().SubQuery()
 
 	// not managed region
 	q1 := CloudregionManager.Query("id").Equals("provider", api.CLOUD_PROVIDER_ONECLOUD)
 
 	// managed region
 	q2 := cloudproviderregions.Query(cloudproviderregions.Field("cloudregion_id", "id"))
-	q2 = q2.Join(providers, sqlchemy.Equals(providers.Field("id"), cloudproviderregions.Field("cloudprovider_id")))
-	q2 = q2.Join(accounts, sqlchemy.Equals(providers.Field("cloudaccount_id"), accounts.Field("id")))
-	q2 = q2.Filter(sqlchemy.OR(
-		sqlchemy.AND(
-			sqlchemy.Equals(providers.Field("domain_id"), domainId),
-			sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
-		),
-		sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
-		sqlchemy.AND(
-			sqlchemy.Equals(accounts.Field("domain_id"), domainId),
-			sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
-		),
-	))
+	providerIds := CloudproviderManager.Query("id")
+	providerIds = CloudproviderManager.filterByDomainId(providerIds, domainId)
+	providersIdsQ := providerIds.Distinct().SubQuery()
+	q2 = q2.Join(providersIdsQ, sqlchemy.Equals(providersIdsQ.Field("id"), cloudproviderregions.Field("cloudprovider_id")))
 
 	return sqlchemy.Union(q1, q2).Query().SubQuery()
 }
@@ -937,3 +926,19 @@ func (manager *SCloudregionManager) FetchDefaultRegion() *SCloudregion {
 func (self *SCloudregion) GetCloudEnv() string {
 	return cloudprovider.GetProviderCloudEnv(self.Provider)
 }
+
+func (manager *SCloudregionManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, manager, "sync-skus")
+}
+
+func (manager *SCloudregionManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.CloudregionSkuSyncInput) (jsonutils.JSONObject, error) {
+	return PerformActionSyncSkus(ctx, userCred, input.Resource, input.SkuSyncInput)
+}
+
+func (manager *SCloudregionManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks")
+}
+
+func (manager *SCloudregionManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) {
+	return GetPropertySkusSyncTasks(ctx, userCred, query)
+}

pkg/compute/models/cloudsync.go

@@ -145,8 +145,8 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca
 
 	if cnt == 0 {
 		// 提前同步instance type.如果同步失败可能导致vm 内存显示为0
-		if err = SyncServerSkusByRegion(ctx, userCred, localRegion); err != nil {
-			msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), err)
+		if ret := SyncServerSkusByRegion(ctx, userCred, localRegion, nil); ret.IsError() {
+			msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), ret.Result())
 			log.Errorln(msg)
 			// 暂时不终止同步
 			// logSyncFailed(provider, task, msg)

pkg/compute/models/dbinstance_skus.go

@@ -628,3 +628,19 @@ func (manager *SDBInstanceSkuManager) ListItemExportKeys(ctx context.Context,
 
 	return q, nil
 }
+
+func (manager *SDBInstanceSkuManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, manager, "sync-skus")
+}
+
+func (manager *SDBInstanceSkuManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SkuSyncInput) (jsonutils.JSONObject, error) {
+	return PerformActionSyncSkus(ctx, userCred, manager.Keyword(), input)
+}
+
+func (manager *SDBInstanceSkuManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks")
+}
+
+func (manager *SDBInstanceSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) {
+	return GetPropertySkusSyncTasks(ctx, userCred, query)
+}

pkg/compute/models/domainquota.go

@@ -191,13 +191,13 @@ func (used *SDomainQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) err
 	sreq := request.(*SDomainQuota)
 	squota := quota.(*SDomainQuota)
 	if quotas.Exceed(used.Globalvpc, sreq.Globalvpc, squota.Globalvpc) {
-		err.Add("globalvpc", squota.Globalvpc, used.Globalvpc, sreq.Globalvpc)
+		err.Add(used, "globalvpc", squota.Globalvpc, used.Globalvpc, sreq.Globalvpc)
 	}
 	if quotas.Exceed(used.Cloudaccount, sreq.Cloudaccount, squota.Cloudaccount) {
-		err.Add("cloudaccount", squota.Cloudaccount, used.Cloudaccount, sreq.Cloudaccount)
+		err.Add(used, "cloudaccount", squota.Cloudaccount, used.Cloudaccount, sreq.Cloudaccount)
 	}
 	if quotas.Exceed(used.DnsZone, sreq.DnsZone, squota.DnsZone) {
-		err.Add("dns_zone", squota.DnsZone, used.DnsZone, sreq.DnsZone)
+		err.Add(used, "dns_zone", squota.DnsZone, used.DnsZone, sreq.DnsZone)
 	}
 	if err.IsError() {
 		return err

pkg/compute/models/elasticcache_skus.go

@@ -649,3 +649,19 @@ func (manager *SElasticcacheSkuManager) ListItemExportKeys(ctx context.Context,
 
 	return q, nil
 }
+
+func (manager *SElasticcacheSkuManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, manager, "sync-skus")
+}
+
+func (manager *SElasticcacheSkuManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SkuSyncInput) (jsonutils.JSONObject, error) {
+	return PerformActionSyncSkus(ctx, userCred, manager.Keyword(), input)
+}
+
+func (manager *SElasticcacheSkuManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks")
+}
+
+func (manager *SElasticcacheSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) {
+	return GetPropertySkusSyncTasks(ctx, userCred, query)
+}

pkg/compute/models/elasticipresource.go

@@ -0,0 +1,23 @@
+package models
+
+import "yunion.io/x/onecloud/pkg/httperrors"
+
+type IGetVpc interface {
+	GetName() string
+	GetVpc() (*SVpc, error)
+}
+
+func ValidateAssociateEip(obj IGetVpc) error {
+	vpc, err := obj.GetVpc()
+	if err != nil {
+		return httperrors.NewGeneralError(err)
+	}
+
+	if vpc != nil {
+		if !vpc.IsSupportAssociateEip() {
+			return httperrors.NewNotSupportedError("resource %s in vpc %s external access mode %s is not support accociate eip", obj.GetName(), vpc.GetName(), vpc.ExternalAccessMode)
+		}
+	}
+
+	return nil
+}

pkg/compute/models/elasticips.go

@@ -85,7 +85,7 @@ type SElasticip struct {
 	Mode string `width:"32" charset:"ascii" get:"user" list:"user" create:"optional"`
 
 	// IP地址
-	IpAddr string `width:"17" charset:"ascii" list:"user" create:"optional"`
+	IpAddr string `width:"17" charset:"ascii" list:"user"`
 
 	// 绑定资源类型
 	AssociateType string `width:"32" charset:"ascii" list:"user"`
@@ -800,14 +800,17 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 		}
 		return input, httperrors.NewResourceNotFoundError2("cloudregion", input.CloudregionId)
 	}
-	region := obj.(*SCloudregion)
+	var (
+		region       = obj.(*SCloudregion)
+		regionDriver = region.GetDriver()
+	)
 	input.CloudregionId = region.GetId()
 
 	// publicIp cannot be created standalone
 	input.Mode = api.EIP_MODE_STANDALONE_EIP
 
-	if len(input.ChargeType) == 0 {
-		input.ChargeType = api.EIP_CHARGE_TYPE_DEFAULT
+	if input.ChargeType == "" {
+		input.ChargeType = regionDriver.GetEipDefaultChargeType()
 	}
 
 	if !utils.IsInStringArray(input.ChargeType, []string{api.EIP_CHARGE_TYPE_BY_BANDWIDTH, api.EIP_CHARGE_TYPE_BY_TRAFFIC}) {
@@ -819,7 +822,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 		return input, err
 	}
 
-	err = region.GetDriver().ValidateCreateEipData(ctx, userCred, &input)
+	err = regionDriver.ValidateCreateEipData(ctx, userCred, &input)
 	if err != nil {
 		return input, err
 	}
@@ -980,6 +983,11 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient.
 		return nil, httperrors.NewInvalidStatusError("cannot associate server in status %s", server.Status)
 	}
 
+	err = ValidateAssociateEip(server)
+	if err != nil {
+		return nil, err
+	}
+
 	if len(self.NetworkId) > 0 {
 		gns, err := server.GetNetworks("")
 		if err != nil {
@@ -1219,9 +1227,13 @@ func (self *SElasticip) getMoreDetails(out api.ElasticipDetails) api.ElasticipDe
 func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCred mcclient.TokenCredential, vm *SGuest,
 	host *SHost, bw int, chargeType string, autoDellocate bool, pendingUsage quotas.IQuota) (*SElasticip, error) {
 	region := host.GetRegion()
+	regionDriver := region.GetDriver()
 
-	if len(chargeType) == 0 {
-		chargeType = api.EIP_CHARGE_TYPE_BY_TRAFFIC
+	if chargeType == "" {
+		chargeType = regionDriver.GetEipDefaultChargeType()
+	}
+	if err := regionDriver.ValidateEipChargeType(chargeType); err != nil {
+		return nil, err
 	}
 
 	eip := &SElasticip{}
@@ -1239,7 +1251,9 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 	eip.ManagerId = host.ManagerId
 	eip.CloudregionId = region.Id
 	eip.Name = fmt.Sprintf("eip-for-%s", vm.GetName())
+
 	if host.ManagerId == "" {
+
 		hostq := HostManager.Query().SubQuery()
 		wireq := WireManager.Query().SubQuery()
 		hostwireq := HostwireManager.Query().SubQuery()
@@ -1299,6 +1313,11 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 }
 
 func (eip *SElasticip) AllocateAndAssociateVM(ctx context.Context, userCred mcclient.TokenCredential, vm *SGuest, parentTaskId string) error {
+	err := ValidateAssociateEip(vm)
+	if err != nil {
+		return err
+	}
+
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(vm.ExternalId), "instance_external_id")
 	params.Add(jsonutils.NewString(vm.Id), "instance_id")

pkg/compute/models/guest_actions.go

@@ -788,6 +788,16 @@ func (self *SGuest) AllowPerformStart(ctx context.Context,
 func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
 	data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_START_FAILED, api.VM_SAVE_DISK_FAILED, api.VM_SUSPEND}) {
+		if !self.guestDisksStorageTypeIsShared() {
+			host := self.GetHost()
+			guestsMem, err := host.GetNotReadyGuestsMemorySize()
+			if err != nil {
+				return nil, err
+			}
+			if float32(guestsMem+self.VmemSize) > host.GetVirtualMemorySize() {
+				return nil, httperrors.NewInsufficientResourceError("host virtual memory not enough")
+			}
+		}
 		if self.isAllDisksReady() {
 			var kwargs *jsonutils.JSONDict
 			if data != nil {
@@ -2112,6 +2122,9 @@ func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.
 	if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("Cannot detach network in status %s", self.Status)
 	}
+	if err := self.GetDriver().ValidateDetachNetwork(ctx, userCred, self); err != nil {
+		return nil, err
+	}
 	var (
 		reserve   = jsonutils.QueryBoolean(data, "reserve", false)
 		netStr, _ = data.GetString("net_id")
@@ -2184,7 +2197,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		if err != nil {
 			return nil, err
 		}
-		if IsExitNetworkInfo(input.Nets[i]) {
+		if IsExitNetworkInfo(userCred, input.Nets[i]) {
 			enicCnt = count
 			// ebw = input.BwLimit
 		} else {
@@ -2742,7 +2755,7 @@ func (self *SGuest) PerformStatus(ctx context.Context, userCred mcclient.TokenCr
 	}
 
 	status := input.Status
-	if len(self.BackupHostId) > 0 && status == api.VM_RUNNING {
+	if len(self.BackupHostId) > 0 && status == api.VM_RUNNING && input.BlockJobsCount > 0 {
 		if len(self.GetMetadata("__mirror_job_status", userCred)) == 0 {
 			self.SetMetadata(ctx, "__mirror_job_status", "ready", userCred)
 		}
@@ -2783,6 +2796,27 @@ func (self *SGuest) PerformStop(ctx context.Context, userCred mcclient.TokenCred
 	}
 }
 
+func (self *SGuest) PerformFreeze(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformFreezeInput) (jsonutils.JSONObject, error) {
+	if self.Freezed {
+		return nil, httperrors.NewBadRequestError("virtual resource already freezed")
+	}
+	if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_STOP_FAILED}) {
+		return nil, self.StartGuestStopAndFreezeTask(ctx, userCred)
+	} else {
+		return self.SVirtualResourceBase.PerformFreeze(ctx, userCred, query, input)
+	}
+}
+
+func (self *SGuest) StartGuestStopAndFreezeTask(ctx context.Context, userCred mcclient.TokenCredential) error {
+	self.SetStatus(userCred, api.VM_START_STOP, "")
+	task, err := taskman.TaskManager.NewTask(ctx, "GuestStopAndFreezeTask", self, userCred, nil, "", "", nil)
+	if err != nil {
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}
+
 func (self *SGuest) AllowPerformRestart(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
@@ -2883,14 +2917,16 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 		return nil, httperrors.NewInvalidStatusError("cannot associate eip in status %s", self.Status)
 	}
 
-	eip, err := self.GetEipOrPublicIp()
+	err := ValidateAssociateEip(self)
+	if err != nil {
+		return nil, err
+	}
+
+	err = self.IsEipAssociable()
 	if err != nil {
-		log.Errorf("Fail to get Eip %s", err)
 		return nil, httperrors.NewGeneralError(err)
 	}
-	if eip != nil {
-		return nil, httperrors.NewInvalidStatusError("already associate with eip")
-	}
+
 	eipStr := input.EipId
 	if len(eipStr) == 0 {
 		return nil, httperrors.NewMissingParameterError("eip_id")
@@ -2904,7 +2940,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 		}
 	}
 
-	eip = eipObj.(*SElasticip)
+	eip := eipObj.(*SElasticip)
 	eipRegion := eip.GetRegion()
 	instRegion := self.getRegion()
 
@@ -2998,10 +3034,24 @@ func (self *SGuest) AllowPerformCreateEip(ctx context.Context, userCred mcclient
 }
 
 func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	var bw int64
-	chargeType, _ := data.GetString("charge_type")
-	if len(chargeType) == 0 {
-		chargeType = api.EIP_CHARGE_TYPE_DEFAULT
+	var (
+		host         = self.GetHost()
+		region       = host.GetRegion()
+		regionDriver = region.GetDriver()
+
+		bw            int64
+		chargeType    string
+		autoDellocate bool
+	)
+
+	err := ValidateAssociateEip(self)
+	if err != nil {
+		return nil, err
+	}
+
+	chargeType, _ = data.GetString("charge_type")
+	if chargeType == "" {
+		chargeType = regionDriver.GetEipDefaultChargeType()
 	}
 
 	bw, _ = data.Int("bandwidth")
@@ -3010,26 +3060,9 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke
 			return nil, httperrors.NewMissingParameterError("bandwidth")
 		}
 	}
-	autoDellocate, _ := data.Bool("auto_dellocate")
+	autoDellocate, _ = data.Bool("auto_dellocate")
 
-	host := self.GetHost()
-	if host == nil {
-		return nil, httperrors.NewInvalidStatusError("No host???")
-	}
-	{
-		if self.ExternalId != "" {
-			_, err := host.GetDriver()
-			if err != nil {
-				return nil, httperrors.NewInvalidStatusError("No valid cloud provider")
-			}
-		}
-		region := host.GetRegion()
-		if region == nil {
-			return nil, httperrors.NewInvalidStatusError("No cloudregion???")
-		}
-	}
-
-	err := self.GetDriver().ValidateCreateEip(ctx, userCred, data)
+	err = self.GetDriver().ValidateCreateEip(ctx, userCred, data)
 	if err != nil {
 		return nil, err
 	}
@@ -4440,6 +4473,9 @@ func (self *SGuest) validateForBatchMigrate(ctx context.Context, rescueMode bool
 	if len(guest.BackupHostId) > 0 {
 		return guest, httperrors.NewBadRequestError("guest %s has backup, can't migrate", guest.Name)
 	}
+	if len(guest.GetIsolatedDevices()) > 0 {
+		return guest, httperrors.NewBadRequestError("guest %s has isolated device, can't migrate", guest.Name)
+	}
 	if rescueMode {
 		if !guest.guestDisksStorageTypeIsShared() {
 			return guest, httperrors.NewBadRequestError("can't rescue geust %s with local storage", guest.Name)

pkg/compute/models/guest_queries.go

@@ -93,6 +93,10 @@ func (manager *SGuestManager) FetchCustomizeColumns(
 					if len(fields) == 0 || fields.Contains("vpc_id") {
 						rows[i].VpcId = strings.Join(gvpc.VpcId, ",")
 					}
+
+					if len(fields) == 0 || fields.Contains("external_access_mode") {
+						rows[i].VpcExternalAccessMode = strings.Join(gvpc.ExternalAccessMode, ",")
+					}
 				}
 			}
 		}
@@ -317,9 +321,10 @@ func (self *SGuest) GetRealIPs() []string {
 }
 
 type sGuestVpcsInfo struct {
-	GuestId string
-	Vpc     []string
-	VpcId   []string
+	GuestId            string
+	Vpc                []string
+	VpcId              []string
+	ExternalAccessMode []string
 }
 
 func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo {
@@ -328,7 +333,7 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo {
 	networks := NetworkManager.Query().SubQuery()
 	guestnetworks := GuestnetworkManager.Query().SubQuery()
 
-	q := vpcs.Query(guestnetworks.Field("guest_id"), vpcs.Field("id"), vpcs.Field("name"))
+	q := vpcs.Query(guestnetworks.Field("guest_id"), vpcs.Field("id"), vpcs.Field("name"), vpcs.Field("external_access_mode"))
 	q = q.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
 	q = q.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
 	q = q.Join(guestnetworks, sqlchemy.Equals(networks.Field("id"), guestnetworks.Field("network_id")))
@@ -336,9 +341,10 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo {
 	q = q.Distinct()
 
 	type sGuestVpcInfo struct {
-		GuestId string
-		Id      string
-		Name    string
+		GuestId            string
+		Id                 string
+		Name               string
+		ExternalAccessMode string
 	}
 	gvpcs := make([]sGuestVpcInfo, 0)
 	err := q.All(&gvpcs)
@@ -351,13 +357,15 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo {
 		gvpc, ok := ret[gvpcs[i].GuestId]
 		if !ok {
 			gvpc = sGuestVpcsInfo{
-				GuestId: gvpcs[i].GuestId,
-				Vpc:     make([]string, 0),
-				VpcId:   make([]string, 0),
+				GuestId:            gvpcs[i].GuestId,
+				Vpc:                make([]string, 0),
+				VpcId:              make([]string, 0),
+				ExternalAccessMode: make([]string, 0),
 			}
 		}
 		gvpc.VpcId = append(gvpc.VpcId, gvpcs[i].Id)
 		gvpc.Vpc = append(gvpc.Vpc, gvpcs[i].Name)
+		gvpc.ExternalAccessMode = append(gvpc.ExternalAccessMode, gvpcs[i].ExternalAccessMode)
 		ret[gvpcs[i].GuestId] = gvpc
 	}
 

pkg/compute/models/guest_template.go

@@ -791,6 +791,7 @@ func (gt *SGuestTemplate) inspect(ctx context.Context, userCred mcclient.TokenCr
 	_, err := GuestTemplateManager.validateContent(ctx, userCred, gt.GetOwnerId(), jsonutils.NewDict(), gt.Content.(*jsonutils.JSONDict))
 	if err == nil {
 		gt.updateCheckTime()
+		gt.SetStatus(userCred, computeapis.GT_READY, "inspect successfully")
 		logclient.AddSimpleActionLog(gt, logclient.ACT_HEALTH_CHECK, "", userCred, true)
 		return nil
 	}
@@ -804,7 +805,7 @@ func (gt *SGuestTemplate) inspect(ctx context.Context, userCred mcclient.TokenCr
 
 func (gm *SGuestTemplateManager) InspectAllTemplate(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 	lastCheckTime := time.Now().Add(time.Duration(-options.Options.GuestTemplateCheckInterval) * time.Hour)
-	q := gm.Query().Equals("status", computeapis.GT_READY)
+	q := gm.Query()
 	q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(q.Field("last_check_time")), sqlchemy.LE(q.Field("last_check_time"),
 		lastCheckTime)))
 	gts := make([]SGuestTemplate, 0, 10)

pkg/compute/models/guestdisks.go

@@ -209,7 +209,8 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 		localpath := disk.GetPathAtHost(host)
 		if len(localpath) == 0 {
 			desc.Add(jsonutils.JSONTrue, "migrating")
-			disk.SetStatus(nil, api.DISK_START_MIGRATE, "migration")
+			// not used yet
+			// disk.SetStatus(nil, api.DISK_START_MIGRATE, "migration")
 		} else {
 			desc.Add(jsonutils.NewString(localpath), "path")
 		}

pkg/compute/models/guestdrivers.go

@@ -95,6 +95,7 @@ type IGuestDriver interface {
 	GetGuestInitialStateAfterCreate() string
 	GetGuestInitialStateAfterRebuild() string
 	GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string
+	GetInstanceCapability() cloudprovider.SInstanceCapability
 
 	OnGuestDeployTaskDataReceived(ctx context.Context, guest *SGuest, task taskman.ITask, data jsonutils.JSONObject) error
 
@@ -141,6 +142,7 @@ type IGuestDriver interface {
 	IsNeedRestartForResetLoginInfo() bool
 	IsRebuildRootSupportChangeImage() bool
 	IsRebuildRootSupportChangeUEFI() bool
+	ValidateDetachNetwork(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error
 
 	IsSupportdDcryptPasswordFromSecretKey() bool
 

pkg/compute/models/guestnetworks.go

@@ -489,7 +489,7 @@ func (self *SGuestnetwork) getJsonDesc(network *SNetwork) *jsonutils.JSONDict {
 	desc.Add(jsonutils.NewString(self.Driver), "driver")
 	desc.Add(jsonutils.NewInt(int64(network.VlanId)), "vlan")
 	desc.Add(jsonutils.NewInt(int64(self.getBandwidth())), "bw")
-	desc.Add(jsonutils.NewInt(int64(self.getMtu())), "mtu")
+	desc.Add(jsonutils.NewInt(int64(self.getMtu(network))), "mtu")
 	desc.Add(jsonutils.NewInt(int64(self.Index)), "index")
 	vips := self.GetVirtualIPs()
 	if len(vips) > 0 {
@@ -722,15 +722,8 @@ func (self *SGuestnetwork) getBandwidth() int {
 	}
 }
 
-func (self *SGuestnetwork) getMtu() int {
-	net := self.GetNetwork()
-	if net != nil {
-		wire := net.GetWire()
-		if wire != nil {
-			return wire.Mtu
-		}
-	}
-	return options.Options.DefaultMtu
+func (self *SGuestnetwork) getMtu(net *SNetwork) int {
+	return net.getMtu()
 }
 
 func (self *SGuestnetwork) IsAllocated() bool {

pkg/compute/models/guests.go

@@ -32,7 +32,6 @@ import (
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/osprofile"
 	"yunion.io/x/pkg/util/regutils"
-	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
@@ -1716,7 +1715,7 @@ func getGuestResourceRequirements(
 	eBw := 0
 	iBw := 0
 	for _, netConfig := range input.Networks {
-		if IsExitNetworkInfo(netConfig) {
+		if IsExitNetworkInfo(userCred, netConfig) {
 			eNicCnt += 1
 			eBw += netConfig.BwLimit
 		} else {
@@ -2325,31 +2324,6 @@ func (self *SGuest) getAdminSecgroupName() string {
 	return ""
 }
 
-func (self *SGuest) GetSecRules() []secrules.SecurityRule {
-	return self.getSecRules()
-}
-
-func (self *SGuest) getSecRules() []secrules.SecurityRule {
-	if secgrp := self.getSecgroup(); secgrp != nil {
-		return secgrp.GetSecRules("")
-	}
-	if rule, err := secrules.ParseSecurityRule(options.Options.DefaultSecurityRules); err == nil {
-		return []secrules.SecurityRule{*rule}
-	} else {
-		log.Errorf("Default SecurityRules error: %v", err)
-	}
-	return []secrules.SecurityRule{}
-}
-
-func (self *SGuest) getSecurityRules() string {
-	secgrp := self.getSecgroup()
-	if secgrp != nil {
-		return secgrp.getSecurityRuleString("")
-	} else {
-		return options.Options.DefaultSecurityRules
-	}
-}
-
 //获取多个安全组规则，优先级降序排序
 func (self *SGuest) getSecurityGroupsRules() string {
 	secgroups, _ := self.GetSecgroups()
@@ -2374,7 +2348,8 @@ func (self *SGuest) getSecurityGroupsRules() string {
 func (self *SGuest) getAdminSecurityRules() string {
 	secgrp := self.getAdminSecgroup()
 	if secgrp != nil {
-		return secgrp.getSecurityRuleString("")
+		ret, _ := secgrp.getSecurityRuleString()
+		return ret
 	} else {
 		return options.Options.DefaultAdminSecurityRules
 	}
@@ -2712,12 +2687,26 @@ func (self *SGuest) getAttach2NetworkCount(net *SNetwork) (int, error) {
 	return q.CountWithError()
 }
 
-func (self *SGuest) getMaxNicIndex() int8 {
+func (self *SGuest) getUsableNicIndex() int8 {
 	nics, err := self.GetNetworks("")
 	if err != nil {
 		return -1
 	}
-	return int8(len(nics))
+	maxIndex := int8(len(nics))
+	for i := int8(0); i <= maxIndex; i++ {
+		found := true
+		for j := range nics {
+			if nics[j].Index == i {
+				found = false
+				break
+			}
+		}
+		if found {
+			return i
+		}
+	}
+	panic(fmt.Sprintf("cannot find usable nic index for guest %s(%s)",
+		self.Name, self.Id))
 }
 
 func (self *SGuest) setOSProfile(ctx context.Context, userCred mcclient.TokenCredential, profile jsonutils.JSONObject) error {
@@ -2795,7 +2784,7 @@ func (self *SGuest) attach2NetworkOnce(
 		return nil, fmt.Errorf("Guest has been attached to network %s", network.Name)
 	}*/
 	if nicConf.Index < 0 {
-		nicConf.Index = self.getMaxNicIndex()
+		nicConf.Index = self.getUsableNicIndex()
 	}
 	if len(driver) == 0 {
 		osProf := self.GetOSProfile()
@@ -4654,7 +4643,7 @@ func (self *SGuestManager) switchBackupGuests(ctx context.Context, userCred mccl
 		log.Errorf("ReconcileBackupGuests failed fetch guests %s", err)
 		return
 	}
-	log.Infof("Guests count %d need reconcile with switch bakcup", len(guests))
+	log.Debugf("Guests count %d need reconcile with switch backup", len(guests))
 	for i := 0; i < len(guests); i++ {
 		val := guests[i].GetMetadataJson("switch_backup", userCred)
 		t, err := val.GetTime()
@@ -4744,6 +4733,28 @@ func (self *SGuest) isInReconcile(userCred mcclient.TokenCredential) bool {
 	return false
 }
 
+func (self *SGuest) IsEipAssociable() error {
+	var eip *SElasticip
+	var err error
+	switch self.Hypervisor {
+	case api.HYPERVISOR_AWS:
+		eip, err = self.GetElasticIp()
+	default:
+		eip, err = self.GetEipOrPublicIp()
+	}
+
+	if err != nil {
+		log.Errorf("Fail to get Eip %s", err)
+		return errors.Wrap(err, "IsEipAssociable")
+	}
+
+	if eip != nil {
+		return httperrors.NewInvalidStatusError("already associate with eip")
+	}
+
+	return nil
+}
+
 func (self *SGuest) GetEipOrPublicIp() (*SElasticip, error) {
 	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id, "")
 }
@@ -4759,10 +4770,16 @@ func (self *SGuest) GetPublicIp() (*SElasticip, error) {
 func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extEip cloudprovider.ICloudEIP, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult {
 	result := compare.SyncResult{}
 
-	eip, err := self.GetEipOrPublicIp()
+	eip, err := self.GetPublicIp()
 	if err != nil {
-		result.Error(fmt.Errorf("getEip error %s", err))
+		result.Error(fmt.Errorf("getPublicIp error %s", err))
 		return result
+	} else if eip == nil {
+		eip, err = self.GetElasticIp()
+		if err != nil {
+			result.Error(fmt.Errorf("getEip error %s", err))
+			return result
+		}
 	}
 
 	if eip == nil && extEip == nil {
@@ -5163,9 +5180,6 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 	// clean some of user input
 	userInput.GenerateName = ""
 	userInput.Description = ""
-	userInput.BillingType = ""
-	userInput.BillingCycle = ""
-	userInput.Duration = ""
 	return userInput
 }
 

pkg/compute/models/host_recycle.go

@@ -344,7 +344,10 @@ func (self *SHost) PerformUndoPrepaidRecycle(ctx context.Context, userCred mccli
 		return nil, httperrors.NewInvalidStatusError("host is not a prepaid recycle host")
 	}
 
-	guests := self.GetGuests()
+	guests, err := self.GetGuests()
+	if err != nil {
+		return nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetGuests"))
+	}
 
 	if len(guests) == 0 {
 		return nil, httperrors.NewInvalidStatusError("cannot delete a recycle host without active instance")
@@ -362,7 +365,7 @@ func (self *SHost) PerformUndoPrepaidRecycle(ctx context.Context, userCred mccli
 		return nil, httperrors.NewInvalidStatusError("cannot undo a recycle host with pending_deleted guest")
 	}
 
-	err := doUndoPrepaidRecycleLockGuest(ctx, userCred, self, &guests[0])
+	err = doUndoPrepaidRecycleLockGuest(ctx, userCred, self, &guests[0])
 	if err != nil {
 		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_UNDO_RECYCLE_PREPAID, self.GetShortDesc(ctx), userCred, false)
 		return nil, httperrors.NewGeneralError(err)