Merge pull request #11410 from yousong/automated-cherry-pick-of-#11406-upstream-release-3.4

Automated cherry pick of #11406: webconsole: wait zombies

.gitignore

@@ -31,6 +31,7 @@ _output
 /structargtest
 .idea
 .vscode
+intellij.yaml
 GPATH
 GRTAGS
 GTAGS

Makefile

@@ -9,7 +9,7 @@ BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
 
 ifeq ($(ONECLOUD_CI_BUILD),)
 	GIT_COMMIT := $(shell git rev-parse --short HEAD)
-	GIT_BRANCH := $(shell git name-rev --name-only HEAD)
+	GIT_BRANCH := $(shell git branch -r --contains | head -1 | sed -E -e "s%(HEAD ->|origin|upstream)/?%%g" | xargs)
 	GIT_VERSION := $(shell git describe --always --tags --abbrev=14 $(GIT_COMMIT)^{commit})
 	GIT_TREE_STATE := $(shell s=`git status --porcelain 2>/dev/null`; if [ -z "$$s" ]; then echo "clean"; else echo "dirty"; fi)
 	BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')

build/monitor/root/opt/yunion/share/notify_templates/alerter/content@cn/DEFAULT

@@ -1,18 +0,0 @@
-## {{.title}}
-- 时间: {{.start_time}}
-- 级别: {{.level}}
-- 详情地址: {{.web_url}}/commonalerts
-
-{{ range .matches}}
-- 指标: {{.metric}}
-- 触发值: {{html .value_str}}
-
-### 触发条件:
-- {{html $.description}}
-
-### 标签
- > 名称: {{ index .tags "name" }}
- > ip: {{ index .tags "ip" }}
- > 平台: {{ index .tags "brand" }}
-------
-{{- end}}

build/monitor/root/opt/yunion/share/notify_templates/alerter/content@en/DEFAULT

@@ -1,18 +0,0 @@
-## {{.title}}
-- Time: {{.start_time}}
-- Level: {{.level}}
-- WebUrl: {{.web_url}}/commonalerts
-
-{{ range .matches}}
-- Metric: {{.metric}}
-- Trigger value: {{html .value_str}}
-
-### Trigger condition:
-- {{html $.description }}
-
-### Label
- > Name: {{ index .tags "name" }}
- > IP: {{ index .tags "ip" }}
- > Brand: {{ index .tags "brand" }}
-------
-{{- end}}

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT

@@ -0,0 +1,9 @@
+{{.title}}
+策略名称: {{.name}}
+触发时间: {{.start_time}}
+报警级别: {{.level}}
+触发条件: {{.description | unescaped}}
+资源数量：{{len .matches}}
+资源名称：{{.resource_name}}
+
+详情地址: {{.web_url}}/commonalerts

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT

@@ -0,0 +1,9 @@
+{{.title}}
+AlertName: {{.name}}
+Time: {{.start_time}}
+Level: {{.level}}
+TriggerCondition: {{.description | unescaped}}
+ResourceCount: {{len .matches}}
+ResourceName: {{.resource_name}}
+
+WebUrl: {{.web_url}}/commonalerts

build/notify/root/opt/yunion/share/template/title/VERIFY.email

@@ -1 +1 @@
-Yunion Verify
+{{.copyright}} Verify

cmd/ansibleserver/main.go

@@ -15,9 +15,14 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/ansibleserver/service"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/baremetal-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/baremetal/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/climc/shell/compute/cloudregions.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -26,6 +27,23 @@ import (
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.Cloudregions).WithKeyword("cloud-region")
+	cmd.PerformClass("sync-skus", &options.CloudregionSkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "cloud-region-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.Cloudregions.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudregionListOptions struct {
 		options.BaseListOptions
 

cmd/climc/shell/compute/dbinstance_skus.go

@@ -15,12 +15,30 @@
 package compute
 
 import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.DBInstanceSkus).WithKeyword("dbinstance-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "dbinstance-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.DBInstanceSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type DBInstanceSkuListOption struct {
 		options.BaseListOptions
 		Engine        string
@@ -54,5 +72,4 @@ func init() {
 		printObject(result)
 		return nil
 	})
-
 }

cmd/climc/shell/compute/elasticcache.go

@@ -19,12 +19,30 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ElasticcacheSkus).WithKeyword("elastic-cache-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "elastic-cache-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ElasticcacheSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.BaseListOptions{}, "elastic-cache-list", "List elastisc cache instance", func(s *mcclient.ClientSession, opts *options.BaseListOptions) error {
 		params, err := options.ListStructToParams(opts)
 		if err != nil {

cmd/climc/shell/compute/elasticips.go

@@ -50,7 +50,7 @@ func init() {
 		Manager    string `help:"cloud provider"`
 		Region     string `help:"cloud region in which EIP is allocated"`
 		Bandwidth  int    `help:"Bandwidth in Mbps"`
-		Ip         string `help:"IP address of the EIP"`
+		IpAddr     string `help:"IP address of the EIP" json:"ip_addr"`
 		Network    string `help:"Network of the EIP"`
 		ChargeType string `help:"bandwidth charge type" choices:"traffic|bandwidth"`
 	}
@@ -75,8 +75,8 @@ func init() {
 			params.Add(jsonutils.NewString(args.Network), "network")
 		}
 
-		if len(args.Ip) > 0 {
-			params.Add(jsonutils.NewString(args.Ip), "ip")
+		if len(args.IpAddr) > 0 {
+			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
 
 		result, err := modules.Elasticips.Create(s, params)

cmd/climc/shell/compute/hosts.go

@@ -407,12 +407,14 @@ func init() {
 	})
 
 	type HostAddNetIfOptions struct {
-		ID     string `help:"ID or Name of host"`
-		WIRE   string `help:"ID or Name of wire to attach"`
-		MAC    string `help:"Mac address of NIC"`
-		INDEX  int64  `help:"nic index"`
-		Type   string `help:"Nic type" choices:"admin|ipmi"`
-		IpAddr string `help:"IP address"`
+		ID        string `help:"ID or Name of host"`
+		WIRE      string `help:"ID or Name of wire to attach"`
+		MAC       string `help:"Mac address of NIC"`
+		INDEX     int64  `help:"nic index"`
+		Type      string `help:"Nic type" choices:"admin|ipmi"`
+		IpAddr    string `help:"IP address"`
+		Bridge    string `help:"Bridge of hostwire"`
+		Interface string `help:"Interface name, eg:eth0, en0"`
 	}
 	R(&HostAddNetIfOptions{}, "host-add-netif", "Host add a NIC", func(s *mcclient.ClientSession, args *HostAddNetIfOptions) error {
 		params := jsonutils.NewDict()
@@ -426,6 +428,12 @@ func init() {
 		if len(args.IpAddr) > 0 {
 			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
+		if len(args.Bridge) > 0 {
+			params.Add(jsonutils.NewString(args.Bridge), "bridge")
+		}
+		if len(args.Interface) > 0 {
+			params.Add(jsonutils.NewString(args.Interface), "interface")
+		}
 		result, err := modules.Hosts.PerformAction(s, args.ID, "add-netif", params)
 		if err != nil {
 			return err

cmd/climc/shell/compute/quotas.go

@@ -316,7 +316,7 @@ func init() {
 		QuotaSetBaseOptions
 		IdentityQuotaOptions
 	}
-	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *InfrasQuotaSetOptions) error {
+	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *IdentityQuotaSetOptions) error {
 		params := jsonutils.Marshal(args)
 		quotas, e := modules.IdentityQuotas.DoQuotaSet(s, params)
 		if e != nil {

cmd/climc/shell/compute/schedulers.go

@@ -123,9 +123,10 @@ func init() {
 		})
 
 	type SchedulerHistoryListOptions struct {
-		Limit  int  `default:"50" help:"Page limit"`
-		Offset int  `default:"0" help:"Page offset"`
-		All    bool `help:"Show all histories, including scheduler-test"`
+		Limit        int  `default:"50" help:"Page limit"`
+		Offset       int  `default:"0" help:"Page offset"`
+		All          bool `help:"Show all histories, including scheduler-test"`
+		IsSuggestion bool `help:"Only show forcast suggestion history"`
 	}
 	R(&SchedulerHistoryListOptions{}, "scheduler-history-list", "Show scheduler history list",
 		func(s *mcclient.ClientSession, args *SchedulerHistoryListOptions) error {
@@ -138,17 +139,14 @@ func init() {
 			if args.Offset > 0 {
 				params.Add(jsonutils.NewInt(int64(args.Offset)), "offset")
 			}
-			if args.All {
-				params.Add(jsonutils.JSONTrue, "all")
-			} else {
-				params.Add(jsonutils.JSONFalse, "all")
-			}
+			params.Add(jsonutils.NewBool(args.All), "all")
+			params.Add(jsonutils.NewBool(args.IsSuggestion), "is_suggestion")
 			result, err := modules.SchedManager.HistoryList(s, params)
 			if err != nil {
 				return err
 			}
 			printList(modulebase.JSON2ListResult(result), []string{
-				"session_id", "time", "status", "consuming",
+				"session_id", "time", "status", "consuming", "is_suggestion",
 			})
 			return nil
 		})

cmd/climc/shell/compute/secgroups.go

@@ -34,4 +34,5 @@ func init() {
 	cmd.Perform("cache-secgroup", &options.SecurityGroupCacheOptions{})
 	cmd.Perform("uncache-secgroup", &options.SecurityGroupUncacheSecurityGroup{})
 	cmd.Perform("purge", &options.SecgroupIdOptions{})
+	cmd.Perform("change-owner", &options.SecgroupChangeOwnerOptions{})
 }

cmd/climc/shell/compute/skus.go

@@ -17,12 +17,30 @@ package compute
 import (
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.ServerSkus).WithKeyword("server-sku")
+	cmd.PerformClass("sync-skus", &options.SkuSyncOptions{})
+
+	R(&options.SkuTaskQueryOptions{}, "server-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.ServerSkus.Get(s, "sync-tasks", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type ServerSkusListOptions struct {
 		options.BaseListOptions
 		Cloudregion    string  `help:"region Id or name"`

cmd/climc/shell/compute/storages.go

@@ -61,13 +61,14 @@ func init() {
 		ID                    string  `help:"ID or Name of storage to update"`
 		Name                  string  `help:"New Name of storage"`
 		Desc                  string  `help:"Description"`
-		CommitBound           float64 `help:"Upper bound of storage overcommit rate"`
+		CommitBound           float64 `help:"Upper bound of storage overcommit rate" json:"cmtbound"`
 		MediumType            string  `help:"Medium type" choices:"ssd|rotate"`
 		RbdRadosMonOpTimeout  int64   `help:"ceph rados_mon_op_timeout"`
 		RbdRadosOsdOpTimeout  int64   `help:"ceph rados_osd_op_timeout"`
 		RbdClientMountTimeout int64   `help:"ceph client_mount_timeout"`
 		RbdKey                string  `help:"ceph rbd key"`
 		Reserved              string  `help:"Reserved storage space"`
+		Capacity              int     `help:"Capacity for storage"`
 	}
 	R(&StorageUpdateOptions{}, "storage-update", "Update a storage", func(s *mcclient.ClientSession, args *StorageUpdateOptions) error {
 		params, err := options.StructToParams(args)

cmd/climc/shell/compute/webconsole.go

@@ -42,34 +42,24 @@ func init() {
 		if err != nil {
 			return err
 		}
-		connParams, err := obj.GetString("connect_params")
-		if err != nil {
+
+		resp := &webconsole_api.ServerRemoteConsoleResponse{}
+		if err := obj.Unmarshal(resp); err != nil {
 			return err
 		}
-		if decodeStr, err := base64.StdEncoding.DecodeString(connParams); err == nil {
-			connParams = string(decodeStr)
-		}
-		var query url.Values
-		connUrl, err := url.ParseRequestURI(connParams)
-		if err == nil {
-			query = connUrl.Query()
-		} else {
-			query, err = url.ParseQuery(connParams)
-			if err != nil {
-				return err
-			}
-		}
-		protocol := query.Get("protocol")
-		if !utils.IsInStringArray(protocol, []string{
+		connectParams := resp.GetConnectParams()
+		if protocol, err := resp.GetConnectProtocol(); err != nil {
+			return err
+		} else if !utils.IsInStringArray(protocol, []string{
 			command.PROTOCOL_TTY, webconsole_api.VNC,
 			webconsole_api.SPICE, webconsole_api.WMKS,
 		}) {
-			fmt.Println(connParams)
+			fmt.Println(connectParams)
 			return nil
 		}
 
-		newQuery := url.Values(make(map[string][]string))
-		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connParams)))
+		newQuery := url.Values{}
+		newQuery.Set("data", base64.StdEncoding.EncodeToString([]byte(connectParams)))
 		u.RawQuery = newQuery.Encode()
 		fmt.Println(u.String())
 		return nil

cmd/climc/shell/helper.go

@@ -284,7 +284,10 @@ type IPerformOpt interface {
 func (cmd ResourceCmd) Perform(action string, args IPerformOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IPerformOpt) error {
-		params := jsonutils.Marshal(args) // .Params()
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
 		ret, err := man.(modulebase.Manager).PerformAction(s, args.GetId(), action, params)
 		if err != nil {
 			return err

cmd/climc/shell/identity/policies.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -183,6 +184,7 @@ func init() {
 	type PolicyShowOptions struct {
 		ID     string `help:"ID of policy"`
 		Format string `help:"policy format, default yaml" default:"yaml" choices:"yaml|json"`
+		Save   string `help:"save policy data into a file"`
 	}
 	R(&PolicyShowOptions{}, "policy-show", "Show policy", func(s *mcclient.ClientSession, args *PolicyShowOptions) error {
 		query := jsonutils.NewDict()
@@ -192,6 +194,15 @@ func init() {
 			return err
 		}
 		printObject(result)
+		if len(args.Save) > 0 {
+			if args.Format == "yaml" {
+				yaml, _ := result.GetString("policy")
+				fileutils2.FilePutContents(args.Save, yaml, false)
+			} else {
+				p, _ := result.Get("policy")
+				fileutils2.FilePutContents(args.Save, p.PrettyString(), false)
+			}
+		}
 		return nil
 	})
 

cmd/climc/shell/misc/copyright.go

@@ -36,6 +36,8 @@ func init() {
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
 		License   *string `help:"the license generator website address"`
+		BrandCn   *string `help:"the brand name in Chinese"`
+		BrandEn   *string `help:"the brand name in English"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -52,6 +54,14 @@ func init() {
 			params.Add(jsonutils.NewString(*args.License), "license")
 		}
 
+		if args.BrandCn != nil {
+			params.Add(jsonutils.NewString(*args.BrandCn), "brand_cn")
+		}
+
+		if args.BrandEn != nil {
+			params.Add(jsonutils.NewString(*args.BrandEn), "brand_en")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

cmd/esxi-agent/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/esxi/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.New().StartService()
 }

cmd/glance/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/image/service"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

cmd/host-deployer/main.go

@@ -14,9 +14,16 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployserver"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
 
 func main() {
+	go procutils.WaitZombieLoop(context.TODO())
+
 	deployer := deployserver.NewDeployService()
 	deployer.StartService()
 }

cmd/host/main.go

@@ -15,14 +15,19 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/service"
 	"yunion.io/x/onecloud/pkg/hostman"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	var srv = &hostman.SHostService{}
 	srv.SServiceBase = &service.SServiceBase{
 		Service: srv,

cmd/vpcagent/main.go

@@ -26,6 +26,7 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/vpcagent/options"
 	_ "yunion.io/x/onecloud/pkg/vpcagent/ovn"
 	"yunion.io/x/onecloud/pkg/vpcagent/worker"
@@ -52,8 +53,12 @@ func main() {
 	}
 
 	{
+
+		ctx := context.Background()
+		ctx, cancelFunc := context.WithCancel(ctx)
+		go procutils.WaitZombieLoop(ctx)
+
 		wg := &sync.WaitGroup{}
-		ctx, cancelFunc := context.WithCancel(context.Background())
 		ctx = context.WithValue(ctx, "wg", wg)
 		wg.Add(1)
 		go w.Start(ctx)

cmd/webconsole/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

go.mod

@@ -141,12 +141,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
+	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
+	yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
+	yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -1124,8 +1124,10 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6 h1:RPpnTSCZwa9VQuWAf7S2dRjnZlYTIM0NH/j5vvUpk9g=
-yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201105032201-9d0fba742954 h1:gGU3uGxw82voqh3tVrTY4q43lLLooqgF662PqOeEpKY=
+yunion.io/x/jsonutils v0.0.0-20201105032201-9d0fba742954/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
@@ -1137,9 +1139,11 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc h1:cgwWI6k9LsXpq6yiOuEIprsXmPyPGUa60sOZu57e8Jo=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8 h1:AkPrqO37Yi3aICys9qDjTlSdVVGh2pWQ8ET+dSJmhoI=
-yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b h1:xBAzDPvvmsOBjxAj0PLHMNLVdPNOlYFHAm7T3IZ2ehs=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -156,6 +156,9 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 			}
 		}
 		resp.Add(domains, "domains")
+		resp.Add(jsonutils.JSONTrue, "return_full_domains")
+	} else {
+		resp.Add(jsonutils.JSONFalse, "return_full_domains")
 	}
 
 	filters := jsonutils.NewDict()
@@ -962,6 +965,12 @@ func getUserInfo2(s *mcclient.ClientSession, uid string, pid string, loginIp str
 		data.Add(jsonutils.JSONFalse, "non_default_domain_projects")
 	}
 
+	if options.Options.EnableQuotaCheck {
+		data.Add(jsonutils.JSONTrue, "enable_quota_check")
+	} else {
+		data.Add(jsonutils.JSONFalse, "enable_quota_check")
+	}
+
 	data.Add(jsonutils.NewString(getSsoCallbackUrl()), "sso_callback_url")
 
 	return data, nil

pkg/apigateway/handler/misc.go

@@ -383,7 +383,9 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			return
 		} else {
 			names[name+"/"+domainId] = true
-			_, err := modules.UsersV3.Get(s, name, nil)
+			params := jsonutils.NewDict()
+			params.Set("domain_id", jsonutils.NewString(domainId))
+			_, err := modules.UsersV3.Get(s, name, params)
 			if err == nil {
 				continue
 			}

pkg/apis/compute/cloudregions_const.go

@@ -114,6 +114,7 @@ const (
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
 	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
+	CITY_PHOENIX        = "Phoenix"        //菲尼克斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -150,7 +151,8 @@ const (
 	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
 
 	// 巴西
-	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+	CITY_SAO_PAULO      = "Sao Paulo"      //圣保罗
+	CITY_RIO_DE_JANEIRO = "Rio de Janeiro" // 里约热内卢
 
 	// 智利
 	CITY_SANTIAGO = "Santiago" // 圣地亚哥

pkg/apis/compute/dbinstance.go

@@ -24,12 +24,13 @@ type DBInstanceCreateInput struct {
 	apis.VirtualResourceCreateInput
 	DeletePreventableCreateInput
 
-	// Ip子网名称或Id,建议使用Id
+	// Ip子网Id
 	// 谷歌云并不实际使用Ip子网,仅仅通过Ip子网确定Vpc
 	// required: true
-	Network string `json:"network"`
+	NetworkId string `json:"network_id"`
+
 	// swagger:ignore
-	NetworkId string
+	Network string `json:"network" yunion-deprecated-by:"network_id"`
 
 	// Ip子网内的地址,不填则按照ip子网的地址分配策略分配一个ip
 	// required: false
@@ -66,8 +67,10 @@ type DBInstanceCreateInput struct {
 	// swagger:ignore
 	CloudregionId string
 
+	VpcId string `json:"vpc_id"`
+
 	// swagger:ignore
-	VpcId string
+	Vpc string `json:"vpc" yunion-deprecated-by:"vpc_id"`
 
 	// swagger:ignore
 	ManagerId string
@@ -142,6 +145,9 @@ type DBInstanceCreateInput struct {
 	// required: true
 	DiskSizeGB int `json:"disk_size_gb"`
 
+	// 指定连接端口
+	Port int `json:"port"`
+
 	// rds初始化密码
 	// 阿里云不需要此参数
 	// 华为云会默认创建一个用户,若不传此参数, 则为随机密码
@@ -163,6 +169,9 @@ type DBInstanceCreateInput struct {
 
 	// swagger:ignore
 	Provider string
+
+	// 从备份中创建新实例
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
 }
 
 type SDBInstanceChangeConfigInput struct {

pkg/apis/compute/elasticip.go

@@ -67,6 +67,8 @@ type SElasticipCreateInput struct {
 	// 子网Id
 	// 私有云创建此参数必传,例如Openstack, ZStack
 	NetworkId string `json:"network_id"`
+	// 指定新建EIP的地址
+	IpAddr string `json:"ip_addr"`
 }
 
 type ElasticipDetails struct {

pkg/apis/compute/elasticips_const.go

@@ -42,7 +42,6 @@ const (
 
 	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
 	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
-	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
 )
 
 var (

pkg/apis/compute/guest_const.go

@@ -249,6 +249,8 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 }
 
 const (
+	VM_DEFAULT_WINDOWS_LOGIN_USER     = "Administrator"
+	VM_DEFAULT_LINUX_LOGIN_USER       = "root"
 	VM_AWS_DEFAULT_LOGIN_USER         = "ec2user"
 	VM_AWS_DEFAULT_WINDOWS_LOGIN_USER = "Administrator"
 	VM_AZURE_DEFAULT_LOGIN_USER       = "toor"

pkg/apis/compute/guests.go

@@ -41,6 +41,11 @@ type ServerListInput struct {
 	Baremetal *bool `json:"baremetal"`
 	// 只列出GPU主机
 	Gpu *bool `json:"gpu"`
+	// 只列出还有备份机的主机
+	Backup *bool `json:"bakcup"`
+	// 列出指定类型的主机
+	// enum: normal,gpu,backup
+	ServerType string `json:"server_type"`
 	// 列出管理安全组为指定安全组的主机
 	AdminSecgroup string `json:"admin_security"`
 	// 列出Hypervisor为指定值的主机
@@ -209,6 +214,8 @@ type ServerDetails struct {
 	Vpc string `json:"vpc"`
 	// 归属VPC ID
 	VpcId string `json:"vpc_id"`
+	// Vpc外网访问模式
+	VpcExternalAccessMode string `json:"vpc_external_access_mode"`
 
 	// 关联安全组列表
 	Secgroups []apis.StandaloneShortDesc `json:"secgroups"`
@@ -417,3 +424,16 @@ type ServerRemoteUpdateInput struct {
 	// 是否覆盖替换所有标签
 	ReplaceTags *bool `json:"replace_tags" help:"replace all remote tags"`
 }
+
+type ServerAssociateEipInput struct {
+	// swagger:ignore
+	// Deprecated
+	Eip string `json:"eip" yunion-deprecated-by:"eip_id"`
+	// 弹性公网IP的ID
+	EipId string `json:"eip_id"`
+}
+
+type ServerDissociateEipInput struct {
+	// 是否自动释放
+	AudoDelete *bool `json:"auto_delete"`
+}

pkg/apis/compute/network_const.go

@@ -62,6 +62,7 @@ var (
 		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,
+		CLOUD_PROVIDER_OPENSTACK,
 	}
 )
 

pkg/apis/compute/secgroupcache.go

@@ -18,6 +18,7 @@ import "yunion.io/x/onecloud/pkg/apis"
 
 type SecurityGroupCacheDetails struct {
 	apis.StatusStandaloneResourceDetails
+	apis.ProjectizedResourceInfo
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
@@ -25,4 +26,6 @@ type SecurityGroupCacheDetails struct {
 
 	// 虚拟私有网络名称
 	Vpc string `json:"vpc"`
+
+	ProjectId string `json:"tenant_id"`
 }

pkg/apis/compute/sku_const.go

@@ -74,7 +74,7 @@ type ServerSkuListInput struct {
 	UsableResourceListInput
 
 	// filter sku by memory size in MB
-	MemorySizeMb int `json:"memory_size_mb"`
+	MemorySizeMb []int `json:"memory_size_mb"`
 	// filter sku by CPU core count
 	CpuCoreCount []int `json:"cpu_core_count"`
 
@@ -155,3 +155,25 @@ type DBInstanceSkuListInput struct {
 	Zone2 []string `json:"zone2"`
 	Zone3 []string `json:"zone3"`
 }
+
+type SkuSyncInput struct {
+	// 云平台名称
+	// example: Google
+	Provider string `json:"provider,omitempty"`
+
+	// 区域ID
+	CloudregionIds []string `json:"cloudregion_ids"`
+}
+
+type SkuTaskQueryInput struct {
+	// 异步任务ID
+	TaskIds []string `json:"task_ids"`
+}
+
+type CloudregionSkuSyncInput struct {
+	SkuSyncInput
+
+	// 同步资源类型
+	// choices: serversku|elasticcachesku|dbinstance_sku
+	Resource string `json:"resource"`
+}

pkg/apis/compute/storage_const.go

@@ -55,12 +55,14 @@ const (
 	STORAGE_LOCAL_SSD     = "local_ssd"
 	STORAGE_LOCAL_PRO     = "local_pro"
 	STORAGE_CLOUD_BASIC   = "cloud_basic"
-	STORAGE_CLOUD_PREMIUM = "cloud_premium"
+	STORAGE_CLOUD_PREMIUM = "cloud_premium" //高性能云硬盘
+	STORAGE_CLOUD_HSSD    = "cloud_hssd"    //增强型SSD云硬盘
 
 	// huawei storage type
-	STORAGE_HUAWEI_SSD  = "SSD"  // 超高IO云硬盘
-	STORAGE_HUAWEI_SAS  = "SAS"  // 高IO云硬盘
-	STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘
+	STORAGE_HUAWEI_SSD   = "SSD"   // 超高IO云硬盘
+	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
+	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
+	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/compute/vpcs_ovn.go

@@ -18,10 +18,13 @@ import (
 	"yunion.io/x/pkg/util/netutils"
 )
 
+const VPC_OVN_ENCAP_COST = 58
+
 const (
 	VPC_EXTERNAL_ACCESS_MODE_DISTGW     = "distgw"     // distgw only
 	VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW = "eip-distgw" // eip when available, distgw otherwise
 	VPC_EXTERNAL_ACCESS_MODE_EIP        = "eip"        // eip only
+	VPC_EXTERNAL_ACCESS_MODE_NONE       = "none"       // no external access
 )
 
 var (
@@ -29,6 +32,7 @@ var (
 		VPC_EXTERNAL_ACCESS_MODE_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW,
 		VPC_EXTERNAL_ACCESS_MODE_EIP,
+		VPC_EXTERNAL_ACCESS_MODE_NONE,
 	}
 )
 

pkg/apis/identity/consts.go

@@ -99,6 +99,7 @@ var (
 
 	CommonWhitelistOptionMap = map[string][]string{
 		"default": []string{
+			"enable_quota_check",
 			"default_quota_value",
 			"enable_rbac",
 			"non_default_domain_projects",
@@ -202,3 +203,19 @@ var (
 		},
 	}
 )
+
+func mergeConfigOptionsFrom(opt1, opt2 map[string][]string) map[string][]string {
+	for opt, values := range opt2 {
+		ovalues, _ := opt1[opt]
+		opt1[opt] = append(ovalues, values...)
+	}
+	return opt1
+}
+
+func MergeServiceConfigOptions(opts ...map[string][]string) map[string][]string {
+	ret := make(map[string][]string)
+	for i := range opts {
+		ret = mergeConfigOptionsFrom(ret, opts[i])
+	}
+	return ret
+}

pkg/apis/input.go

@@ -201,6 +201,8 @@ type PerformStatusInput struct {
 	// 更改的目标状态值
 	// required:true
 	Status string `json:"status"`
+	// swagger:ignore
+	BlockJobsCount int `json:"block_jobs_count"`
 
 	// 更改状态的原因描述
 	// required:false

pkg/apis/monitor/alert.go

@@ -117,6 +117,7 @@ type AlertCreateInput struct {
 	NoDataState string `json:"no_data_state"`
 	// 报警执行错误将当前报警状态设置为对应的状态
 	ExecutionErrorState string `json:"execution_error_state"`
+	UsedBy              string `json:"used_by"`
 }
 
 type AlertUpdateInput struct {

pkg/apis/monitor/alertrecord.go

@@ -34,7 +34,9 @@ type AlertRecordCreateInput struct {
 
 type AlertRecordRule struct {
 	Metric          string `json:"metric"`
+	Measurement     string `json:"measurement"`
 	MeasurementDesc string `json:"measurement_desc"`
+	Field           string `json:"field"`
 	FieldDesc       string `json:"field_desc"`
 	// 比较运算符, 比如: >, <, >=, <=
 	Comparator string `json:"comparator"`

pkg/apis/monitor/suggestsysrule.go

@@ -23,6 +23,11 @@ import (
 const (
 	METRIC_TAG   = "TAG"
 	METRIC_FIELD = "FIELD"
+
+	METRIC_VM_ID    = "vm_id"
+	METRIC_OSS_ID   = "oss_id"
+	METRIC_RDS_ID   = "rds_id"
+	METRIC_REDIS_ID = "redis_id"
 )
 
 var PROPERTY_TYPE = []string{"databases", "measurements", "metric-measurement"}

pkg/apis/monitor/template.go

@@ -19,9 +19,10 @@ type NotificationTemplateCreateInput struct {
 }
 
 type NotificationTemplateConfig struct {
-	Title   string      `json:"title"`
-	Name    string      `json:"name"`
-	Matches []EvalMatch `json:"matches"`
+	Title        string      `json:"title"`
+	Name         string      `json:"name"`
+	ResourceName string      `json:"resource_name"`
+	Matches      []EvalMatch `json:"matches"`
 	// PrevAlertState AlertStateType `json:"prev_alert_state"`
 	// State AlertStateType `json:"state"`
 	NoDataFound bool   `json:"no_data"`

pkg/apis/scheduler/api.go

@@ -81,6 +81,10 @@ type ScheduleInput struct {
 	CpuMicrocode string `json:"cpu_microcode"`
 	CpuMode      string `json:"cpu_mode"`
 
+	// In the migrate and create backup cases
+	// we don't need reallocate network
+	ReuseNetwork bool `json:"reuse_network"`
+
 	PendingUsages []jsonutils.JSONObject
 }
 

pkg/apis/webconsole/types.go

@@ -0,0 +1,53 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package webconsole
+
+import (
+	"encoding/base64"
+	"net/url"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type ServerRemoteConsoleResponse struct {
+	ConnectParams string `json:"connect_params"`
+	Session       string `json:"session,omitempty"`
+
+	apis.Meta
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectParams() string {
+	params := resp.ConnectParams
+	if data, err := base64.StdEncoding.DecodeString(params); err == nil {
+		params = string(data)
+	}
+	return params
+}
+
+func (resp *ServerRemoteConsoleResponse) GetConnectProtocol() (string, error) {
+	var (
+		params = resp.GetConnectParams()
+		query  url.Values
+	)
+	if uri, err := url.ParseRequestURI(params); err == nil {
+		query = uri.Query()
+	} else if q, err := url.ParseQuery(params); err == nil {
+		query = q
+	} else {
+		return "", err
+	}
+	protocol := query.Get("protocol")
+	return protocol, nil
+}

pkg/baremetal/cronjobs.go

@@ -93,7 +93,7 @@ func (job *SStatusProbeJob) Name() string {
 
 func (job *SStatusProbeJob) Do(ctx context.Context, now time.Time) error {
 	bStatus := job.baremetal.GetStatus()
-	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING {
+	if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING || bStatus == api.BAREMETAL_UNKNOWN {
 		ps, err := job.baremetal.GetPowerStatus()
 		if err != nil {
 			return errors.Wrap(err, "GetPowerStatus")

pkg/baremetal/pxe/dhcp.go

@@ -368,7 +368,7 @@ func (req *dhcpRequest) createBaremetal(session *mcclient.ClientSession) (jsonut
 	params := jsonutils.NewDict()
 	mac := req.ClientMac.String()
 	zoneId := req.baremetalManager.GetZoneId()
-	name := fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1))
+	name := strings.ToLower(fmt.Sprintf("BM%s", strings.Replace(mac, ":", "", -1)))
 	params.Add(jsonutils.NewString(name), "name")
 	params.Add(jsonutils.NewString(mac), "access_mac")
 	params.Add(jsonutils.NewString("baremetal"), "host_type")

pkg/baremetal/utils/ipmitool/ipmitool.go

@@ -15,6 +15,7 @@
 package ipmitool
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -115,21 +116,21 @@ func (ipmi *LanPlusIPMI) GetMode() string {
 	return "rmcp"
 }
 
-func (ipmi *LanPlusIPMI) GetCommand(args ...string) *procutils.Command {
+func (ipmi *LanPlusIPMI) GetCommand(args ...string) (*procutils.Command, context.CancelFunc) {
 	nArgs := []string{
-		"--signal=KILL",
-		fmt.Sprintf("%s", ipmi.GetDefaultTimeout()),
-		"ipmitool", "-I", "lanplus", "-H", ipmi.host,
+		"-I", "lanplus", "-H", ipmi.host,
 		"-p", fmt.Sprintf("%d", ipmi.port),
 		"-U", ipmi.user,
 		"-P", ipmi.password,
 	}
 	nArgs = append(nArgs, args...)
-	return procutils.NewCommand("timeout", nArgs...)
+	ctx, cancel := context.WithTimeout(context.Background(), ipmi.GetDefaultTimeout())
+	return procutils.NewCommandContext(ctx, "ipmitool", nArgs...), cancel
 }
 
 func (ipmi *LanPlusIPMI) ExecuteCommand(args ...string) ([]string, error) {
-	cmd := ipmi.GetCommand(args...)
+	cmd, cancel := ipmi.GetCommand(args...)
+	defer cancel()
 	log.Debugf("[LanPlusIPMI] execute command: %s", cmd.String())
 	out, err := cmd.Output()
 	if err != nil {

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -624,7 +624,7 @@ func (a StorcliAdaptor) key() string {
 }
 
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.key() != ""
+	return a.Controller >= 0 && a.name != "" && a.sn != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {
@@ -756,7 +756,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build JBOD fail: %v", err)
+	log.Errorf("Try storcli build JBOD fail: %v", err)
 	labels := []string{}
 	for _, dev := range devs {
 		labels = append(labels, GetSpecString(dev))
@@ -779,7 +779,7 @@ func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalSt
 	if err == nil {
 		return nil
 	}
-	log.Errorf("Try build jbod fail: %v", err)
+	log.Errorf("Try megacli build jbod fail: %v", err)
 	cmds := []string{}
 	for _, dev := range devs {
 		cmd := GetCommand("-CfgLdAdd", "-r0", fmt.Sprintf("[%s]", GetSpecString(dev)),
@@ -862,29 +862,42 @@ func (adapter *MegaRaidAdaptor) RemoveLogicVolumes() error {
 	return nil
 }
 
-/*
-def _storcli_clear_jbod_disks(self):
-    cmds = []
-    for dev in self.devs:
-        cmd = self.raid.get_command2(
-                    '/c%d/e%d/s%d' % (self.index, dev.enclosure, dev.slot),
-                    'set', 'good', 'force')
-        cmds.append(cmd)
-    logging.info('%s', cmds)
-    self.raid.term.exec_remote_commands(cmds)
-*/
+func (adapter *MegaRaidAdaptor) storcliClearJBODDisks() error {
+	errs := make([]error, 0)
+	for _, dev := range adapter.devs {
+		cmd := GetCommand2(fmt.Sprintf("/c%d/e%d/s%d", adapter.index, dev.enclosure, dev.slot), "set", "good", "force")
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "Set PD good storcli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
+}
 
 func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
 	devIds := []string{}
 	for idx, dev := range adapter.devs {
 		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage(idx)))
 	}
-	cmd := GetCommand("-PDMakeGood", fmt.Sprintf("-PhysDrv[%s]", strings.Join(devIds, ",")), "-Force", fmt.Sprintf("-a%d", adapter.index))
-	_, err := adapter.remoteRun(cmd)
-	return err
+	errs := make([]error, 0)
+	for _, devId := range devIds {
+		cmd := GetCommand("-PDMakeGood", "-PhysDrv", fmt.Sprintf("'[%s]'", devId), "-Force", fmt.Sprintf("-a%d", adapter.index))
+		if _, err := adapter.remoteRun(cmd); err != nil {
+			err = errors.Wrapf(err, "PDMakeGood megacli cmd %v", cmd)
+			errs = append(errs, err)
+		}
+	}
+	return errors.NewAggregate(errs)
 }
 
 func (adapter *MegaRaidAdaptor) clearJBODDisks() {
+	if err := adapter.megacliClearJBODDisks(); err != nil {
+		log.Errorf("megacliClearJBODDisks error: %v", err)
+		log.Infof("try storcliClearJBODDisks")
+		if err := adapter.storcliClearJBODDisks(); err != nil {
+			log.Errorf("storcliClearJBODDisks error: %v", err)
+		}
+	}
 	adapter.megacliEnableJBOD(true)
 	adapter.megacliEnableJBOD(false)
 	adapter.megacliEnableJBOD(true)

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -56,7 +56,7 @@ func (dev *Mpt2SASRaidPhyDev) parseLine(line string) bool {
 	}
 	switch key {
 	case "Drive Type":
-		if strings.HasSuffix("_HDD", val) {
+		if strings.HasSuffix(val, "_HDD") {
 			dev.Rotate = tristate.True
 		} else {
 			dev.Rotate = tristate.False

pkg/cloudcommon/consts/consts.go

@@ -35,6 +35,8 @@ var (
 	maxPagingLimit     int64 = 2048
 
 	domainizedNamespace = true
+
+	enableQuotaCheck = false
 )
 
 func SetRegion(region string) {
@@ -85,3 +87,11 @@ func SetDomainizedNamespace(domainNS bool) {
 func IsDomainizedNamespace() bool {
 	return nonDefaultDomainProjects && domainizedNamespace
 }
+
+func SetEnableQuotaCheck(val bool) {
+	enableQuotaCheck = val
+}
+
+func EnableQuotaCheck() bool {
+	return enableQuotaCheck
+}

pkg/cloudcommon/db/domainresource.go

@@ -201,6 +201,9 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 		}
 	}
 
+	// cancel usage
+	model.cleanModelUsages(ctx, userCred)
+
 	_, err = Update(model, func() error {
 		model.DomainId = ownerId.GetProjectDomainId()
 		model.DomainSrc = string(apis.OWNER_SOURCE_LOCAL)
@@ -210,6 +213,9 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 		return nil, errors.Wrap(err, "Update")
 	}
 
+	// add usage
+	model.RecoverUsages(ctx, userCred)
+
 	OpsLog.SyncOwner(model, former, userCred)
 	notes := struct {
 		OldDomainId string

pkg/cloudcommon/db/jointbase.go

@@ -34,7 +34,7 @@ import (
 type SJointResourceBase struct {
 	SResourceBase
 
-	RowId int64 `primary:"true" auto_increment:"true"`
+	RowId int64 `primary:"true" auto_increment:"true" list:"user"`
 }
 
 type SJointResourceBaseManager struct {

pkg/cloudcommon/db/metadata.go

@@ -459,17 +459,17 @@ func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key s
 }
 
 func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
 		return err
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
 
-func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
+func (manager *SMetadataManager) setValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
 	// no need to lock
@@ -546,9 +546,9 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 }
 
 func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "setValues")
 	}
 
 	idStr := GetObjectIdstr(obj)
@@ -565,9 +565,9 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 	q := manager.Query().Equals("id", idStr).NotLike("key", `\_\_%`) //避免删除系统内置的metadata, _ 在mysql里面有特殊含义,需要转义
 	switch delRange {
 	case USER_TAG_PREFIX:
-		q = q.Like("key", USER_TAG_PREFIX+"%")
+		q = q.Startswith("key", USER_TAG_PREFIX)
 	case CLOUD_TAG_PREFIX:
-		q = q.Like("key", CLOUD_TAG_PREFIX+"%")
+		q = q.Startswith("key", CLOUD_TAG_PREFIX)
 	}
 	q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys)))
 	if err := FetchModelObjects(manager, q, &records); err != nil {
@@ -581,7 +581,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }

pkg/cloudcommon/db/modelbase.go

@@ -548,17 +548,24 @@ func (model *SModelBase) CustomizeDelete(ctx context.Context, userCred mcclient.
 	return nil
 }
 
-func cleanModelUsages(ctx context.Context, userCred mcclient.TokenCredential, model IModel) {
+func (model *SModelBase) cleanModelUsages(ctx context.Context, userCred mcclient.TokenCredential) {
 	usages := model.GetIModel().GetUsages()
 	if CancelUsages != nil && len(usages) > 0 {
 		CancelUsages(ctx, userCred, usages)
 	}
 }
 
+func (model *SModelBase) RecoverUsages(ctx context.Context, userCred mcclient.TokenCredential) {
+	usages := model.GetIModel().GetUsages()
+	if AddUsages != nil && len(usages) > 0 {
+		AddUsages(ctx, userCred, usages)
+	}
+}
+
 func (model *SModelBase) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
 	// clean usage on predelete
 	// clean usage before fakedelete for pending delete models
-	cleanModelUsages(ctx, userCred, model)
+	model.cleanModelUsages(ctx, userCred)
 }
 
 func (model *SModelBase) PostDelete(ctx context.Context, userCred mcclient.TokenCredential) {

pkg/cloudcommon/db/opslog.go

@@ -117,7 +117,8 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
 	if !consts.OpsLogEnabled() {
 		return
 	}
-	if len(model.GetId()) == 0 || len(model.GetName()) == 0 {
+	if len(model.GetId()) == 0 {
+		log.Errorf("logevent for an object without ID???")
 		return
 	}
 	if action == ACT_UPDATE {

pkg/cloudcommon/db/opslog_const.go

@@ -47,8 +47,9 @@ const (
 	ACT_BACKUP_START        = "backup_start"
 	ACT_BACKUP_START_FAILED = "backup_start_fail"
 
-	ACT_FREEZE   = "freeze"
-	ACT_UNFREEZE = "unfreeze"
+	ACT_FREEZE      = "freeze"
+	ACT_FREEZE_FAIL = "freeze_fail"
+	ACT_UNFREEZE    = "unfreeze"
 
 	ACT_RESTARING    = "restarting"
 	ACT_RESTART_FAIL = "restart_fail"
@@ -260,4 +261,6 @@ const (
 
 	ACT_SYNC_VPCS        = "sync_vpcs"
 	ACT_SYNC_RECORD_SETS = "sync_record_sets"
+
+	ACT_UPDATE_RULE = "update_config"
 )

pkg/cloudcommon/db/orderby.go

@@ -29,7 +29,7 @@ func OrderByFields(q *sqlchemy.SQuery, fieldOrders []string, fields []sqlchemy.I
 	for i := range fields {
 		if sqlchemy.SQL_ORDER_ASC.Equals(fieldOrders[i]) {
 			q = q.Asc(fields[i])
-		} else {
+		} else if sqlchemy.SQL_ORDER_DESC.Equals(fieldOrders[i]) {
 			q = q.Desc(fields[i])
 		}
 	}

pkg/cloudcommon/db/quotas/errors.go

@@ -19,9 +19,11 @@ import (
 	"strings"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SOutOfQuotaError struct {
+	scope   rbacutils.TRbacScope
 	name    string
 	limit   int
 	used    int
@@ -37,7 +39,7 @@ func (e *SOutOfQuotaError) Cause() error {
 }
 
 func (e *SOutOfQuotaError) Error() string {
-	return fmt.Sprintf("%s limit %d used %d request %d", e.name, e.limit, e.used, e.request)
+	return fmt.Sprintf("[%s.%s] limit %d used %d request %d", e.scope, e.name, e.limit, e.used, e.request)
 }
 
 func (es *SOutOfQuotaErrors) Error() string {
@@ -63,8 +65,10 @@ func NewOutOfQuotaError() *SOutOfQuotaErrors {
 	}
 }
 
-func (es *SOutOfQuotaErrors) Add(name string, limit int, used int, request int) {
+func (es *SOutOfQuotaErrors) Add(quota IQuota, name string, limit int, used int, request int) {
+	scope := quota.GetKeys().Scope()
 	e := SOutOfQuotaError{
+		scope:   scope,
 		name:    name,
 		limit:   limit,
 		used:    used,

pkg/cloudcommon/db/quotas/handler.go

@@ -132,6 +132,12 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 	ret := jsonutils.NewDict()
 
 	keys := quota.GetKeys()
+	ret.Update(jsonutils.Marshal(keys))
+	ret.Update(quota.ToJSON(""))
+
+	if !consts.EnableQuotaCheck() {
+		return ret, nil
+	}
 
 	usage := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, usage)
@@ -150,8 +156,6 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 		return nil, errors.Wrap(err, "manager.GetPendingUsages")
 	}
 
-	ret.Update(jsonutils.Marshal(keys))
-	ret.Update(quota.ToJSON(""))
 	if usage != nil {
 		ret.Update(usage.ToJSON("usage"))
 	}

pkg/cloudcommon/db/quotas/interface.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type IQuotaKeys interface {
@@ -30,6 +31,8 @@ type IQuotaKeys interface {
 	Compare(IQuotaKeys) int
 
 	OwnerId() mcclient.IIdentityProvider
+
+	Scope() rbacutils.TRbacScope
 }
 
 type IQuota interface {

pkg/cloudcommon/db/quotas/register.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -57,6 +58,10 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 }
 
 func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	if localUsage == nil {
 		return nil
 	}
@@ -65,6 +70,10 @@ func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential,
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	manager := getQuotaManager(quota)
 	err := manager.checkSetPendingQuota(ctx, userCred, quota)
 	if err != nil {
@@ -75,6 +84,9 @@ func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		cancelUsage(ctx, userCred, usage.(IQuota))
 	}
@@ -89,6 +101,9 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 }
 
 func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		addUsage(ctx, userCred, usage.(IQuota))
 	}

pkg/cloudcommon/db/quotas/usageworker.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -61,6 +62,12 @@ func isDirty(key string) bool {
 }
 
 func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan IQuota, realTime bool) {
+	if !consts.EnableQuotaCheck() {
+		go func() {
+			usageChan <- nil
+		}()
+		return
+	}
 	key := QuotaKeyString(keys)
 	setDirty(key)
 
@@ -105,6 +112,10 @@ func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan I
 }
 
 func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
+
 	log.Infof("CalculateQuotaUsages")
 	quota := manager.newQuota()
 	keys := quota.GetKeys()

pkg/cloudcommon/db/rbac.go

@@ -51,6 +51,8 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeUser:
 		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() || objOwnerId.GetUserId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && objOwnerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}
@@ -103,6 +105,8 @@ func isClassRbacAllowed(manager IModelManager, userCred mcclient.TokenCredential
 	case rbacutils.ScopeUser:
 		if ownerId != nil && ownerId.GetUserId() == objOwnerId.GetUserId() {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}

pkg/cloudcommon/db/sharablebase.go

@@ -472,6 +472,10 @@ func SharablePerformPublic(model ISharableBaseModel, ctx context.Context, userCr
 	if targetScope.HigherThan(allowScope) {
 		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", targetScope, allowScope)
 	}
+	requireScope := model.GetPublicScope()
+	if requireScope.HigherThan(allowScope) {
+		return errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", requireScope, allowScope)
+	}
 
 	_, err = Update(model, func() error {
 		model.SetShare(targetScope)

pkg/cloudcommon/db/taskman/localtaskworker.go

@@ -37,8 +37,8 @@ func Error2TaskData(err error) jsonutils.JSONObject {
 	return errJson
 }
 
-func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
-	localTaskWorkerMan.Run(func() {
+func LocalTaskRunWithWorkers(task ITask, proc func() (jsonutils.JSONObject, error), wm *appsrv.SWorkerManager) {
+	wm.Run(func() {
 
 		log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN STARTXXXXXXXXXXXXXXXXX")
 		defer log.Debugf("XXXXXXXXXXXXXXXXXXLOCAL TASK RUN END  XXXXXXXXXXXXXXXXX")
@@ -59,3 +59,7 @@ func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
 
 	}, nil, nil)
 }
+
+func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) {
+	LocalTaskRunWithWorkers(task, proc, localTaskWorkerMan)
+}

pkg/cloudcommon/db/taskman/tasks.go

@@ -41,6 +41,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
@@ -280,8 +281,9 @@ func (manager *STaskManager) NewTask(
 	}
 	parentTask := task.GetParentTask()
 	if parentTask != nil {
-		st := SSubTask{TaskId: parentTask.Id, Stage: parentTask.Stage, SubtaskId: task.Id}
-		err := SubTaskManager.TableSpec().Insert(ctx, &st)
+		st := &SSubTask{TaskId: parentTask.Id, Stage: parentTask.Stage, SubtaskId: task.Id}
+		st.SetModelManager(SubTaskManager, st)
+		err := SubTaskManager.TableSpec().Insert(ctx, st)
 		if err != nil {
 			log.Errorf("Subtask insert error %s", err)
 			return nil, err
@@ -814,7 +816,11 @@ func (self *STask) GetObjects() []db.IStandaloneModel {
 }
 
 func (task *STask) GetTaskRequestHeader() http.Header {
-	header := mcclient.GetTokenHeaders(task.GetUserCred())
+	userCred := task.GetUserCred()
+	if !userCred.IsValid() {
+		userCred = auth.AdminCredential()
+	}
+	header := mcclient.GetTokenHeaders(userCred)
 	header.Set(mcclient.TASK_ID, task.GetTaskId())
 	return header
 }

pkg/cloudcommon/db/virtualresource.go

@@ -380,6 +380,9 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC
 		}
 	}
 
+	// cancel usage
+	model.cleanModelUsages(ctx, userCred)
+
 	_, err = Update(model, func() error {
 		model.DomainId = ownerId.GetProjectDomainId()
 		model.ProjectId = ownerId.GetProjectId()
@@ -390,6 +393,9 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC
 		return nil, errors.Wrap(err, "Update")
 	}
 
+	// add usage
+	model.RecoverUsages(ctx, userCred)
+
 	OpsLog.SyncOwner(model, former, userCred)
 	notes := struct {
 		OldProjectId string
@@ -457,13 +463,6 @@ func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, user
 	return nil, nil
 }
 
-func (model *SVirtualResourceBase) RecoverUsages(ctx context.Context, userCred mcclient.TokenCredential) {
-	usages := model.GetIModel().GetUsages()
-	if AddUsages != nil && len(usages) > 0 {
-		AddUsages(ctx, userCred, usages)
-	}
-}
-
 func (model *SVirtualResourceBase) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	err := model.CancelPendingDelete(ctx, userCred)
 	if err == nil {

pkg/cloudcommon/etcd/etcd.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
@@ -45,7 +46,8 @@ type SEtcdClient struct {
 	onKeepaliveFailure func()
 	leaseLiving        bool
 
-	watchers map[string]*SEtcdWatcher
+	watchers   map[string]*SEtcdWatcher
+	watchersMu *sync.Mutex
 }
 
 func defaultOnKeepAliveFailed() {
@@ -115,6 +117,7 @@ func NewEtcdClient(opt *SEtcdOptions, onKeepaliveFailure func()) (*SEtcdClient,
 	etcdClient.leaseTtlTimeout = timeoutSeconds
 
 	etcdClient.watchers = make(map[string]*SEtcdWatcher)
+	etcdClient.watchersMu = &sync.Mutex{}
 
 	etcdClient.namespace = opt.EtcdNamspace
 
@@ -143,6 +146,10 @@ func (cli *SEtcdClient) GetClient() *clientv3.Client {
 	return cli.client
 }
 
+func (cli *SEtcdClient) SessionLiving() bool {
+	return cli.leaseLiving
+}
+
 func (cli *SEtcdClient) startSession() error {
 	ctx := context.TODO()
 
@@ -302,8 +309,10 @@ func (cli *SEtcdClient) Watch(
 	onModify TEtcdModifyEventFunc,
 	onDelete TEtcdDeleteEventFunc,
 ) error {
+	cli.watchersMu.Lock()
 	_, ok := cli.watchers[prefix]
 	if ok {
+		cli.watchersMu.Unlock()
 		return errors.Errorf("watch prefix %s already registered", prefix)
 	}
 
@@ -314,6 +323,7 @@ func (cli *SEtcdClient) Watch(
 		watcher: watcher,
 		cancel:  cancel,
 	}
+	cli.watchersMu.Unlock()
 
 	prefix = cli.getKey(prefix)
 
@@ -342,6 +352,8 @@ func (cli *SEtcdClient) Watch(
 }
 
 func (cli *SEtcdClient) Unwatch(prefix string) {
+	cli.watchersMu.Lock()
+	defer cli.watchersMu.Unlock()
 	watcher, ok := cli.watchers[prefix]
 	if ok {
 		log.Debugf("unwatch %s", prefix)

pkg/cloudcommon/informer/etcd.go

@@ -156,6 +156,10 @@ func (b *EtcdBackend) put(ctx context.Context, key, val string) error {
 	return b.PutWithLease(ctx, key, val, b.leaseTTL)
 }
 
+func (b *EtcdBackend) PutSession(ctx context.Context, key, val string) error {
+	return b.client.PutSession(ctx, key, val)
+}
+
 func (b *EtcdBackend) PutWithLease(ctx context.Context, key, val string, ttlSeconds int64) error {
 	return b.client.PutWithLease(ctx, key, val, ttlSeconds)
 }

pkg/cloudcommon/informer/etcd_client.go

@@ -99,7 +99,7 @@ func (b *EtcdBackendForClient) registerClientResource(ctx context.Context, key s
 	if err != nil {
 		return err
 	}
-	if err := b.PutWithLease(ctx, clientKey, "ok", 60); err != nil {
+	if err := b.PutSession(ctx, clientKey, "ok"); err != nil {
 		return err
 	}
 	return nil

pkg/cloudcommon/informer/informer.go

@@ -92,7 +92,7 @@ func Create(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Create(ctx, obj)
 	})
 }
@@ -101,7 +101,7 @@ func Update(ctx context.Context, obj *ModelObject, oldObj *jsonutils.JSONDict) e
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Update(ctx, obj, oldObj)
 	})
 }
@@ -110,7 +110,7 @@ func Delete(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Delete(ctx, obj)
 	})
 }

pkg/cloudcommon/informer/worker.go

@@ -15,6 +15,8 @@
 package informer
 
 import (
+	"context"
+
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -29,14 +31,35 @@ func init() {
 	informerWorkerMan = appsrv.NewWorkerManager("InformerWorkerManager", 10, 10240, false)
 }
 
-func run(f func(be IInformerBackend) error) error {
+/*type noCancel struct {
+	ctx context.Context
+}
+
+func (c noCancel) Deadline() (time.Time, bool) {
+	return time.Time{}, false
+}
+
+func (c noCancel) Done() <-chan struct{} {
+	return nil
+}
+
+func (c noCancel) Err() error {
+	return nil
+}
+
+func (c noCancel) Value(key interface{}) interface{} {
+	return c.ctx.Value(key)
+}*/
+
+func run(ctx context.Context, f func(ctx context.Context, be IInformerBackend) error) error {
 	be := GetDefaultBackend()
 	if be == nil {
 		return ErrBackendNotInit
 	}
 	wf := func() {
 		nopanic.Run(func() {
-			if err := f(be); err != nil {
+			// outside context ignored cause of run in worker
+			if err := f(context.Background(), be); err != nil {
 				log.Errorf("run informer error: %v", err)
 			}
 		})

pkg/cloudcommon/notifyclient/notify.go

@@ -91,7 +91,9 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 		if err != nil {
 			return nil, err
 		}
-		tmp, err := template.New(key).Parse(string(cont))
+		tmp := template.New(key)
+		tmp.Funcs(template.FuncMap{"unescaped": unescaped})
+		tmp, err = tmp.Parse(string(cont))
 		if err != nil {
 			return nil, err
 		}
@@ -100,6 +102,10 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 	return templatesTable[key], nil
 }
 
+func unescaped(str string) template.HTML {
+	return template.HTML(str)
+}
+
 func getContent(ctx context.Context, topic string, contType string, channel npk.TNotifyChannel, data jsonutils.JSONObject) (string, error) {
 	if channel == npk.NotifyByWebhook {
 		return "", nil
@@ -332,7 +338,13 @@ func notifyRobot(ctx context.Context, robot string, recipientId []string, isGrou
 	if err != nil {
 		return err
 	}
-	jarray, _ := result.Get("types")
+	if result == nil {
+		return nil
+	}
+	jarray, err := result.Get("types")
+	if err != nil {
+		return err
+	}
 	cTypes := jarray.(*jsonutils.JSONArray).GetStringArray()
 	for _, ct := range cTypes {
 		RawNotifyWithCtx(ctx, recipientId, isGroup, npk.TNotifyChannel(ct), priority, event, data)

pkg/cloudcommon/options/changes.go

@@ -40,6 +40,9 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 		consts.SetDomainizedNamespace(newOpts.DomainizedNamespace)
 		changed = true
 	}
+	if oldOpts.EnableQuotaCheck != newOpts.EnableQuotaCheck {
+		consts.SetEnableQuotaCheck(newOpts.EnableQuotaCheck)
+	}
 	return changed
 }
 

pkg/cloudcommon/options/manager.go

@@ -124,7 +124,10 @@ func (manager *SOptionManager) DoSync(first bool) (time.Duration, error) {
 			appsrv.SetExitFlag()
 		}
 		copyOptions(manager.options, newOpts)
+		// if first {
+		// upload config
 		manager.session.Upload()
+		// }
 	}
 	return manager.refreshInterval, nil
 }

pkg/cloudcommon/options/mergeconf.go

@@ -66,6 +66,8 @@ type mcclientServiceConfigSession struct {
 	session   *mcclient.ClientSession
 	serviceId string
 	config    *jsonutils.JSONDict
+
+	commonServiceId string
 }
 
 func newServiceConfigSession() IServiceConfigSession {
@@ -88,12 +90,12 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 			merged = true
 		} else {
 			// not initialized
-			s.Upload()
+			// s.Upload()
 		}
 	}
-	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
-	if len(commonServiceId) > 0 {
-		commonConf, err := getServiceConfig(s.session, commonServiceId)
+	s.commonServiceId, _ = getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
+	if len(s.commonServiceId) > 0 {
+		commonConf, err := getServiceConfig(s.session, s.commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
 		} else if commonConf != nil {
@@ -123,6 +125,11 @@ func (s *mcclientServiceConfigSession) Upload() {
 			// ignore the error
 			log.Errorf("fail to save config: %s", err)
 		}
+		_, err = modules.ServicesV3.PerformAction(s.session, s.commonServiceId, "config", nconf)
+		if err != nil {
+			// ignore the error
+			log.Errorf("fail to save common config: %s", err)
+		}
 	}
 }
 

pkg/cloudcommon/options/options.go

@@ -78,6 +78,7 @@ type BaseOptions struct {
 	IsSlaveNode        bool `help:"Slave mode"`
 	CronJobWorkerCount int  `help:"Cron job worker count" default:"4"`
 
+	EnableQuotaCheck  bool   `help:"enable quota check" default:"true"`
 	DefaultQuotaValue string `help:"default quota value" choices:"unlimit|zero|default" default:"default"`
 
 	CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"`

pkg/cloudcommon/validators/validators.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/regutils"
@@ -821,3 +822,15 @@ func NewIPv4AddrValidator(key string) *ValidatorIPv4Addr {
 	v.SetParent(v)
 	return v
 }
+
+var ValidateModel = func(userCred mcclient.TokenCredential, manager db.IStandaloneModelManager, id *string) (db.IModel, error) {
+	model, err := manager.FetchByIdOrName(userCred, *id)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), *id)
+		}
+		return nil, httperrors.NewGeneralError(err)
+	}
+	*id = model.GetId()
+	return model, nil
+}

pkg/cloudevent/models/cloudproviders.go

@@ -262,23 +262,12 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	if err != nil {
 		return start, end, errors.Wrap(err, "self.GetProviderFactory")
 	}
-	q := CloudeventManager.Query().Equals("cloudprovider_id", self.Id).Desc("created_at")
-	count, err := q.CountWithError()
-	if err != nil {
-		return start, end, errors.Wrap(err, "q.CountWithError")
-	}
 	if !self.LastSyncTimeAt.IsZero() {
 		start = self.LastSyncTimeAt
-	} else if count == 0 {
-		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	} else {
-		event := &SCloudevent{}
-		err = q.First(event)
-		if err != nil {
-			return start, end, errors.Wrap(err, "q.First")
-		}
-		start = event.CreatedAt
+		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	}
+
 	// 避免cloudevent过长时间未运行，再次运行时记录的最后一条时间距离现在间隔太长
 	if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
 		start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)

pkg/cloudevent/service/handlers.go

@@ -55,6 +55,8 @@ func InitHandlers(app *appsrv.Application) {
 
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
+		db.Metadata,
+
 		proxy.ProxySettingManager,
 		models.CloudeventManager,
 	} {

pkg/cloudid/models/cloudaccount.go

@@ -168,6 +168,7 @@ func (manager *SCloudaccountManager) syncCloudaccounts(ctx context.Context, user
 			continue
 		}
 		account.StartSyncCloudIdResourcesTask(ctx, userCred, "")
+		account.StartSystemCloudpolicySyncTask(ctx, userCred, false, "") // 避免新加账号未能及时同步策略
 		localAccounts = append(localAccounts, *account)
 		result.Add()
 	}
@@ -676,7 +677,16 @@ func (self *SCloudaccount) GetCloudaccountByProvider(provider string) ([]SClouda
 	return accounts, nil
 }
 
-func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential) error {
+func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error {
+	dbPolicies, err := self.GetSystemCloudpolicies()
+	if err != nil {
+		return errors.Wrapf(err, "GetSystemCloudpolicies")
+	}
+
+	if len(dbPolicies) > 0 && !refresh {
+		return nil
+	}
+
 	factory, err := self.GetProviderFactory()
 	if err != nil {
 		return errors.Wrapf(err, "GetProviderFactory")
@@ -716,22 +726,18 @@ func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context,
 			return errors.Wrapf(err, "GetISystemCloudpolicies for account %s(%s)", self.Name, self.Provider)
 		}
 	}
-	return self.syncSystemCloudpoliciesFromCloud(ctx, userCred, iPolicies)
+	return self.syncSystemCloudpoliciesFromCloud(ctx, userCred, iPolicies, dbPolicies)
 }
 
-func (self *SCloudaccount) syncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, iPolicies []cloudprovider.ICloudpolicy) error {
+func (self *SCloudaccount) syncSystemCloudpoliciesFromCloud(ctx context.Context, userCred mcclient.TokenCredential, iPolicies []cloudprovider.ICloudpolicy, dbPolicies []SCloudpolicy) error {
 	result := compare.SyncResult{}
-	dbPolicies, err := self.GetSystemCloudpolicies()
-	if err != nil {
-		return errors.Wrapf(err, "GetSystemCloudpolicies")
-	}
 
 	removed := make([]SCloudpolicy, 0)
 	commondb := make([]SCloudpolicy, 0)
 	commonext := make([]cloudprovider.ICloudpolicy, 0)
 	added := make([]cloudprovider.ICloudpolicy, 0)
 
-	err = compare.CompareSets(dbPolicies, iPolicies, &removed, &commondb, &commonext, &added)
+	err := compare.CompareSets(dbPolicies, iPolicies, &removed, &commondb, &commonext, &added)
 	if err != nil {
 		return errors.Wrapf(err, "compare.CompareSets")
 	}
@@ -832,6 +838,9 @@ func (self *SCloudaccount) GetICloudprovider() ([]SCloudprovider, error) {
 }
 
 func (self *SCloudaccount) syncCloudprovider(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
+	lockman.LockRawObject(ctx, "cloudproviders", self.Id)
+	defer lockman.ReleaseRawObject(ctx, "cloudproviders", self.Id)
+
 	result := compare.SyncResult{}
 
 	providers, err := self.GetICloudprovider()
@@ -932,22 +941,28 @@ func (manager *SCloudaccountManager) SyncCloudidSystemPolicies(ctx context.Conte
 		log.Errorf("GetSupportCloudIdAccounts error: %v", err)
 		return
 	}
+	providers := []string{}
 	for i := range accounts {
 		_, err := accounts[i].GetProvider() // 检查账号是否可以正常连接
 		if err != nil {
 			log.Errorf("GetProvider for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err)
 			continue
 		}
-		err = accounts[i].StartSystemCloudpolicySyncTask(ctx, userCred, "")
+		if utils.IsInStringArray(accounts[i].Provider, providers) {
+			continue
+		}
+		err = accounts[i].StartSystemCloudpolicySyncTask(ctx, userCred, true, "")
 		if err != nil {
 			log.Errorf("StartSystemCloudpolicySyncTask for account %s(%s) error: %v", accounts[i].Name, accounts[i].Provider, err)
 			continue
 		}
+		providers = append(providers, accounts[i].Provider)
 	}
 }
 
-func (self *SCloudaccount) StartSystemCloudpolicySyncTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+func (self *SCloudaccount) StartSystemCloudpolicySyncTask(ctx context.Context, userCred mcclient.TokenCredential, refresh bool, parentTaskId string) error {
 	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewBool(refresh), "refresh")
 	task, err := taskman.TaskManager.NewTask(ctx, "SystemCloudpolicySyncTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
 		return errors.Wrap(err, "NewTask")

pkg/cloudid/tasks/sync_cloudid_system_policies_task.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudid/models"
 )
@@ -39,7 +40,12 @@ func (self *SystemCloudpolicySyncTask) taskFailed(ctx context.Context, cloudacco
 
 func (self *SystemCloudpolicySyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
 	account := obj.(*models.SCloudaccount)
-	err := account.SyncSystemCloudpoliciesFromCloud(ctx, self.GetUserCred())
+
+	lockman.LockRawObject(ctx, "system policies", account.Provider)
+	defer lockman.ReleaseRawObject(ctx, "system policies", account.Provider)
+
+	refresh := jsonutils.QueryBoolean(self.GetParams(), "refresh", false)
+	err := account.SyncSystemCloudpoliciesFromCloud(ctx, self.GetUserCred(), refresh)
 	if err != nil {
 		self.taskFailed(ctx, account, errors.Wrapf(err, "SyncSystemCloudpoliciesFromCloud"))
 		return

pkg/cloudprovider/dbinstance.go

@@ -16,6 +16,14 @@ package cloudprovider
 
 import "yunion.io/x/onecloud/pkg/util/billing"
 
+type TBackupMethod string
+
+const (
+	BackupMethodLogical  = TBackupMethod("Logical")
+	BackupMethodPhysical = TBackupMethod("Physical")
+	BackupMethodUnknown  = TBackupMethod("")
+)
+
 type SDBInstanceNetwork struct {
 	IP        string
 	NetworkId string
@@ -55,6 +63,9 @@ type SManagedDBInstanceCreateConfig struct {
 	ProjectId        string
 
 	BillingCycle *billing.SBillingCycle
+
+	// 仅从备份恢复到新实例用到
+	RdsId string
 }
 
 type SManagedDBInstanceChangeConfig struct {

pkg/cloudprovider/instance.go

@@ -26,6 +26,52 @@ import (
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )
 
+type SDistDefaultAccount struct {
+	// 操作系统发行版
+	OsDistribution string
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改
+	Changeable bool
+}
+
+type SOsDefaultAccount struct {
+	// 默认用户名
+	DefaultAccount string
+	// 是否可更改用户名
+	Changeable bool
+	// 禁止使用的账号
+	DisabledAccounts []string
+	// 各操作系统发行版的默认用户名信息
+	DistAccounts []SDistDefaultAccount
+}
+
+type SDefaultAccount struct {
+	Linux   SOsDefaultAccount
+	Windows SOsDefaultAccount
+}
+
+type StorageInfo struct {
+	StorageType string
+	MaxSizeGb   int
+	MinSizeGb   int
+	Resizable   bool
+	StepSizeGb  int
+}
+
+type Storage struct {
+	DataDisk []StorageInfo
+	SysDisk  []StorageInfo
+}
+
+type SInstanceCapability struct {
+	Provider   string
+	Hypervisor string
+	Storages   Storage
+
+	DefaultAccount SDefaultAccount
+}
+
 type SDiskInfo struct {
 	StorageExternalId string
 	StorageType       string