Merge pull request #7486 from wanyaoqi/automated-cherry-pick-of-#7484-upstream-release-3.2

Automated cherry pick of #7484: fix host info get sshkey

build/docker/Dockerfile.ansibleserver

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:1.0.0
 
-ADD ./_output/bin/ansibleserver /opt/yunion/bin/ansibleserver
+ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -1,13 +1,11 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
+FROM alpine:3.11.6
 
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk update && \
-    apk add --no-cache tzdata ansible ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+# openssh-client, for ansible ssh connection
+RUN set -x \
+	&& apk update \
+	&& apk add openssh-client \
+	&& apk add ansible \
+	&& apk add tzdata ca-certificates \
+	&& rm -rf /var/cache/apk/*

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/logger /opt/yunion/bin/logger
 

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/monitor /opt/yunion/bin/monitor
 

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -7,7 +7,7 @@ ENV TZ Asia/Shanghai
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates && \
+    apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.yunionconf

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/yunionconf /opt/yunion/bin/yunionconf
 

build/notify/root/opt/yunion/share/template/content/VERIFY.email

@@ -11,7 +11,7 @@
   			<table style="margin-left: 20px;">
   				<tr>
   					<td>
-  						<img src="data:{{.logo_format}};base64,{{.logo}}" alt="" style="color: #fff; width: 80px; vertical-align: middle;">
+  						<img src="data:{{.login_logo_format}};base64,{{.login_logo}}" alt="" style="color: #fff; height: 32px; vertical-align: middle;">
   					</td>
   				</tr>
   			</table>
@@ -56,10 +56,10 @@
   		</td>
   	</tr>
   	<tr style="width: 96%;">
-		<td colspan="2" style="border-top: 1px dashed #ccc; color: #ccc; font-size: 12px; padding-bottom: 10px; font-weight: 100;">*为了确保您的帐号安全，该链接仅48小时内访问有效，请勿直接回复此邮件。</td>
+		<td colspan="2" style="border-top: 1px dashed #ccc; color: #333; font-size: 12px; padding-bottom: 10px; font-weight: 100;">*为了确保您的帐号安全，该链接仅48小时内访问有效，请勿直接回复此邮件。</td>
 	</tr>
 	<tr>
-		<td colspan="2" style="background: #333; text-align: right; padding-right: 20px; font-size: 12px; color: #949494; height: 50px;">版权所有 © {{.copyright}} 保留一切权利</td>
+		<td colspan="2" style="background: #333; text-align: right; padding-right: 20px; font-size: 12px; color: #fff; height: 50px;">版权所有 © {{.copyright}} 保留一切权利</td>
 	</tr>
   </table>
 </body>

cmd/climc/shell/globalvpcs.go

@@ -15,6 +15,8 @@
 package shell
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -78,4 +80,30 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	R(&GlobalVpcShowOptions{}, "global-vpc-change-owner-candidate-domains", "Show candiate domains of a global vpc for changing owner", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
+		result, err := modules.GlobalVpcs.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type GlobalVpcChangeOwnerOptions struct {
+		ID            string `help:"ID or name of vpc" json:"-"`
+		ProjectDomain string `json:"project_domain" help:"target domain"`
+	}
+	R(&GlobalVpcChangeOwnerOptions{}, "global-vpc-change-owner", "Change owner domain of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcChangeOwnerOptions) error {
+		if len(args.ProjectDomain) == 0 {
+			return fmt.Errorf("empty project_domain")
+		}
+		params := jsonutils.Marshal(args)
+		ret, err := modules.GlobalVpcs.PerformAction(s, args.ID, "change-owner", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/identityproviders.go

@@ -299,7 +299,9 @@ func init() {
 		IDP string `help:"identity provider name or ID"`
 	}
 	R(&IdentityProviderConfigEditOptions{}, "idp-config-edit", "Edit config yaml of an identity provider", func(s *mcclient.ClientSession, args *IdentityProviderConfigEditOptions) error {
-		conf, err := modules.IdentityProviders.GetSpecific(s, args.IDP, "config", nil)
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.JSONTrue, "sensitive")
+		conf, err := modules.IdentityProviders.GetSpecific(s, args.IDP, "config", params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/images.go

@@ -252,7 +252,7 @@ func init() {
 	}
 
 	type ImageDeleteOptions struct {
-		ID                    []string `help:"Image ID or name"`
+		ID                    []string `help:"Image ID or name" json:"-"`
 		OverridePendingDelete *bool    `help:"Delete image directly instead of pending delete" short-token:"f"`
 	}
 	R(&ImageDeleteOptions{}, "image-delete", "Delete a image", func(s *mcclient.ClientSession, args *ImageDeleteOptions) error {

cmd/climc/shell/policies.go

@@ -288,12 +288,30 @@ func init() {
 		if err != nil {
 			log.Fatalf("Set log level %q: %v", "debug", err)
 		}
+		if args.Debug {
+			rbacutils.ShowMatchRuleDebug = true
+		}
 		auth.InitFromClientSession(s)
 		policy.EnableGlobalRbac(15*time.Second, 15*time.Second, false)
 		if args.Debug {
 			consts.EnableRbacDebug()
 		}
 
+		findPolicy := false
+		for !findPolicy {
+			all := policy.PolicyManager.AllPolicies()
+			for _, allP := range all {
+				if len(allP) > 0 {
+					findPolicy = true
+					break
+				}
+			}
+			if findPolicy {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
 		req := jsonutils.NewDict()
 		for i := 0; i < len(args.Request); i += 1 {
 			parts := strings.Split(args.Request[i], ":")
@@ -359,6 +377,7 @@ func init() {
 				Context: mcclient.SAuthContext{
 					Ip: args.Ip,
 				},
+				Token: "faketoken",
 			}
 		} else {
 			token = s.GetToken()
@@ -370,6 +389,10 @@ func init() {
 		}
 		printObject(result)
 
+		for _, r := range args.Role {
+			fmt.Println("role", r, "matched policies:", policy.PolicyManager.RoleMatchPolicies(r))
+		}
+
 		fmt.Println("userCred:", token)
 		for _, scope := range []rbacutils.TRbacScope{
 			rbacutils.ScopeSystem,

cmd/climc/shell/secgrouprules.go

@@ -25,33 +25,20 @@ import (
 func init() {
 	type SecGroupRulesListOptions struct {
 		options.BaseListOptions
-		Secgroup  string `help:"Secgroup ID or Name"`
-		Direction string `help:"filter Direction of rule" choices:"in|out"`
-		Protocol  string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Action    string `help:"filter Actin of rule" choices:"allow|deny"`
+		Secgroup     string   `help:"Secgroup ID or Name"`
+		SecgroupName string   `help:"Search rules by fuzzy secgroup name"`
+		Projects     []string `help:"Filter rules by project"`
+		Direction    string   `help:"filter Direction of rule" choices:"in|out"`
+		Protocol     string   `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
+		Action       string   `help:"filter Actin of rule" choices:"allow|deny"`
+		Ports        string   `help:"filter Ports of rule"`
+		Ip           string   `help:"filter cidr of rule"`
 	}
 
 	R(&SecGroupRulesListOptions{}, "secgroup-rule-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupRulesListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Secgroup) > 0 {
-			params.Add(jsonutils.NewString(args.Secgroup), "secgroup")
-		}
-		if len(args.Direction) > 0 {
-			params.Add(jsonutils.NewString(args.Direction), "direction")
-		}
-		if len(args.Protocol) > 0 {
-			params.Add(jsonutils.NewString(args.Protocol), "protocol")
-		}
-		if len(args.Action) > 0 {
-			params.Add(jsonutils.NewString(args.Action), "action")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroupRules.List(s, params)
 		if err != nil {

cmd/climc/shell/secgroups.go

@@ -29,23 +29,15 @@ func init() {
 		Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"`
 		Server string `help:"Filter secgroups bound to specified server"`
 		options.BaseListOptions
+		Ip        string `help:"Filter secgroup by ip"`
+		Ports     string `help:"Filter secgroup by ports"`
+		Direction string `help:"Filter secgroup by ports" choices:"all|in|out"`
 	}
 
 	R(&SecGroupsListOptions{}, "secgroup-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupsListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Equals) > 0 {
-			params.Add(jsonutils.NewString(args.Equals), "equals")
-		}
-		if len(args.Server) > 0 {
-			params.Add(jsonutils.NewString(args.Server), "server")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroups.List(s, params)
 		if err != nil {

cmd/googlecli/main.go

@@ -18,17 +18,21 @@ import (
 	"fmt"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
 	Debug        bool   `help:"debug mode"`
 	Help         bool   `help:"Show help"`
+	AuthFile     string `help:"google cloud auth json file path" default:"$GOOGLE_AUTH_FILE"`
 	ClientEmail  string `help:"Client email" default:"$GOOGLE_CLIENT_EMAIL"`
 	ProjectID    string `help:"Project ID" default:"$GOOGLE_PROJECT_ID"`
 	PrivateKeyID string `help:"Private Key ID" default:"$GOOGLE_PRIVATE_KEY_ID"`
@@ -78,6 +82,20 @@ func showErrorAndExit(e error) {
 }
 
 func newClient(options *BaseOptions) (*google.SRegion, error) {
+	if len(options.AuthFile) > 0 {
+		jsonStr, err := fileutils2.FileGetContents(options.AuthFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "FileGetContents")
+		}
+		jsonCfg, err := jsonutils.ParseString(jsonStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.ParseString")
+		}
+		options.ClientEmail, _ = jsonCfg.GetString("client_email")
+		options.PrivateKeyID, _ = jsonCfg.GetString("private_key_id")
+		options.PrivateKey, _ = jsonCfg.GetString("private_key")
+		options.ProjectID, _ = jsonCfg.GetString("project_id")
+	}
 	if len(options.ClientEmail) == 0 {
 		return nil, fmt.Errorf("Missing ClientEmail")
 	}

go.mod

@@ -117,6 +117,7 @@ require (
 	google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb
 	google.golang.org/grpc v1.26.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/fatih/set.v0 v0.2.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
@@ -127,12 +128,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
+	yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
-	yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa
+	yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
-	yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e
+	yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11
+	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 
 replace (

go.sum

@@ -1013,6 +1013,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fatih/set.v0 v0.2.1 h1:Xvyyp7LXu34P0ROhCyfXkmQCAoOUKb1E2JS9I7SE5CY=
+gopkg.in/fatih/set.v0 v0.2.1/go.mod h1:5eLWEndGL4zGGemXWrKuts+wTJR0y+w+auqUJZbmyBg=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -1106,8 +1108,8 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501 h1:i1r9XvbdxH3FgTCLmTaRi3MzQqhiQimXJRlUOPgrxnU=
-yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383 h1:XEhTD+FGjBjEIWwcY0Ez1lY3oTTzjUOcRgH/i4eFM5A=
+yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383/go.mod h1:3U5hspYn9apQHaXKY+fJ4Qgwjey32Kc+Mq+mUe9NamQ=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
@@ -1115,12 +1117,13 @@ yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960Ru
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa h1:VizPfW8+mLFEE7W/97zxQJbfdxchi2dn1M/Y7YBwTTc=
-yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b h1:wwPzH8iMRYjtaSM2pBN7nOqeZxR4XW5wdz2VE1KH/Ug=
+yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b h1:Ov/eKwwNeQje7SggkopExzcCW/GfaUouvXkcY6RUEcM=
+yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e h1:pctCe/EPel3F1B83pJ2q9b34Umd1NdbLW1Yd+Lzur2s=
-yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11 h1:srrssomGJLh4OaDaY/1so+2FUBjZxdwQx/1pZ6403YU=
+yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/service/handlers.go

@@ -39,6 +39,7 @@ func InitHandlers(app *appsrv.Application) {
 	db.InitAllManagers()
 
 	db.RegisterModelManager(db.OpsLog)
+	db.RegisterModelManager(db.Metadata)
 	db.RegisterModelManager(db.UserCacheManager)
 	db.RegisterModelManager(db.TenantCacheManager)
 	for _, manager := range []db.IModelManager{

pkg/apigateway/handler/auth.go

@@ -42,6 +42,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/netutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 func AppContextToken(ctx context.Context) mcclient.TokenCredential {
@@ -271,7 +272,7 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 			return nil, httperrors.NewInputParameterError("get password in body")
 		}
 		// try base64 decryption
-		if decPasswd, err := base64.StdEncoding.DecodeString(passwd); err == nil {
+		if decPasswd, err := base64.StdEncoding.DecodeString(passwd); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
 			passwd = string(decPasswd)
 		}
 		if len(uname) == 0 || len(passwd) == 0 {
@@ -395,7 +396,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
+		httperrors.ForbiddenError(w, "user forbidden login from web")
 		return false
 	}
 	return true

pkg/apigateway/handler/misc.go

@@ -28,6 +28,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -57,6 +59,12 @@ const (
 	BATCH_HOST_REGISTER_QUANTITY_LIMITATION = 1000
 )
 
+var (
+	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
+	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+)
+
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
 	token := AppContextToken(ctx)
 	session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion)
@@ -170,20 +178,43 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	// check header line
+	titlesOk := false
+	for _, t := range [][]string{BatchHostRegisterTemplate, BatchHostISORegisterTemplate, BatchHostPXERegisterTemplate} {
+		if len(t) == len(rows[0]) {
+			for _, title := range rows[0] {
+				if !utils.IsInStringArray(title, t) {
+					break
+				}
+			}
+
+			titlesOk = true
+		}
+	}
+
+	if !titlesOk {
+		httperrors.InputParameterError(w, "template file is invalid.please check.")
+		return
+	}
+
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +230,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 
@@ -297,18 +353,6 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			return
 		}
 
-		if _, ok := names[name]; ok {
-			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
-			httperrors.JsonClientError(w, e)
-			return
-		} else {
-			names[name] = true
-			_, err := modules.UsersV3.Get(s, name, nil)
-			if err == nil {
-				continue
-			}
-		}
-
 		domainId, ok := domains[domain]
 		if !ok {
 			if len(domain) == 0 {
@@ -327,6 +371,18 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			domains[domain] = id
 		}
 
+		if _, ok := names[name+"/"+domainId]; ok {
+			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
+			httperrors.JsonClientError(w, e)
+			return
+		} else {
+			names[name+"/"+domainId] = true
+			_, err := modules.UsersV3.Get(s, name, nil)
+			if err == nil {
+				continue
+			}
+		}
+
 		user := jsonutils.NewDict()
 		user.Add(jsonutils.NewString(name), "name")
 		user.Add(jsonutils.NewString(domainId), "domain_id")
@@ -375,21 +431,21 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 	var content bytes.Buffer
 	switch template {
 	case "BatchHostRegister":
-		records := [][]string{{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}}
+		records := [][]string{BatchHostRegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostISORegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}}
+		records := [][]string{BatchHostISORegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostPXERegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}}
+		records := [][]string{BatchHostPXERegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")

pkg/apigateway/options/options.go

@@ -26,7 +26,7 @@ type GatewayOptions struct {
 
 	DisableModuleApiVersion bool `help:"Disable each modules default api version" default:"false"`
 
-	EnableTotp bool `help:"Enable two-factor authentication"  default:"false"`
+	EnableTotp bool `help:"Enable two-factor authentication" default:"false"`
 
 	SqlitePath string `help:"sqlite db path" default:"/etc/yunion/data/yunionapi.db"`
 

pkg/apis/compute/cloudaccount.go

@@ -180,9 +180,9 @@ type CloudaccountCreateInput struct {
 	// 自动同步间隔时间
 	SyncIntervalSeconds int `json:"sync_interval_seconds"`
 
-	// 自动根据云上项目或订阅创建本地项目
+	// 自动根据云上项目或订阅创建本地项目, OpenStack此参数为true
 	// default: false
-	AutoCreateProject bool `json:"auto_create_project"`
+	AutoCreateProject *bool `json:"auto_create_project"`
 
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONDict `json:"options"`

pkg/apis/compute/cloudprovider.go

@@ -251,3 +251,23 @@ type CloudproviderUpdateInput struct {
 
 type CloudproviderCreateInput struct {
 }
+
+type CloudproviderGetStorageClassInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetStorageClassOutput struct {
+	// 对象存储存储类型
+	StorageClasses []string `json:"storage_classes"`
+}
+
+type CloudproviderGetCannedAclInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetCannedAclOutput struct {
+	// Bucket支持的预置ACL列表
+	BucketCannedAcls []string `json:"bucket_canned_acls"`
+	// Object支持的预置ACL列表
+	ObjectCannedAcls []string `json:"object_canned_acls"`
+}

pkg/apis/compute/cloudregions_const.go

@@ -51,7 +51,7 @@ const (
 	CITY_FU_ZHOU       = "Fuzhou"      //福州
 	CITY_WU_HAN        = "Wuhan"       //武汉
 	CITY_CHANG_SHA     = "Changsha"    //长沙
-	CITY_SHU_ZHOU      = "Shuzhou"     //苏州
+	CITY_SU_ZHOU       = "Suzhou"      //苏州
 	CITY_BAO_DING      = "Baoding"     //保定
 	CITY_NAN_JING      = "Nanjing"     //南京
 	CITY_FO_SHAN       = "Foshan"      //佛山

pkg/apis/compute/dbinstance.go

@@ -101,7 +101,7 @@ type DBInstanceCreateInput struct {
 	// | -----		| ------	| --- |
 	// | 华为云		|ha, single, replica| |
 	// | 阿里云		|basic, high_availability, always_on, finance||
-	// | Google		|SECOND_GEN | FIRST_GEN 目前谷歌已弃用|
+	// | Google		|Zonal, Regional | |
 	// 翻译:
 	// basic: 基础版
 	// high_availability: 高可用
@@ -110,7 +110,8 @@ type DBInstanceCreateInput struct {
 	// ha: 高可用
 	// single: 单机
 	// replica: 只读
-	// SECNOD_GEN: 第二代
+	// Zonal: 单区域
+	// Regional: 区域级
 	// required: true
 	Category string `json:"category"`
 
@@ -170,9 +171,22 @@ type SDBInstanceChangeConfigInput struct {
 type SDBInstanceRecoveryConfigInput struct {
 	apis.Meta
 
-	DBInstancebackup   string
-	DBInstancebackupId string            `json:"dbinstancebackup_id"`
-	Databases          map[string]string `json:"databases,allowempty"`
+	// swagger:ignore
+	DBInstancebackup string `json:"dbinstancebackup" "yunion:deprecated-by":"dbinstancebackup_id"`
+
+	// 备份Id
+	//
+	//
+	// | 平台		| 支持引擎								| 说明		|
+	// | -----		| ------								| ---		|
+	// | 华为云		|MySQL, SQL Server						| 仅SQL Server支持恢复到当前实例			|
+	// | 阿里云		|MySQL, SQL Server						| MySQL要求必须开启单库单表恢复功能 并且只能是MySQL 8.0 高可用版（本地SSD盘）MySQL 5.7 高可用版（本地SSD盘）或MySQL 5.6 高可用版, MySQL仅支持恢复到当前实例|
+	// | Google		|MySQL, PostgreSQL, SQL Server			| PostgreSQL备份恢复时，要求实例不能有副本			|
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
+
+	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中, 阿里云此参数为必传
+	// example: {"sdb1":"ddb1"}
+	Databases map[string]string `json:"databases,allowempty"`
 }
 
 type DBInstanceListInput struct {

pkg/apis/compute/dbinstance_account.go

@@ -70,13 +70,13 @@ type SDBInstanceSetPrivilegesInput struct {
 
 type DBInstancePrivilege struct {
 	// 数据库名称
-	Database string
+	Database string `json:"database"`
 	// 账号名称
-	Account string
+	Account string `json:"account"`
 	// 数据库Id
-	DBInstancedatabaseId string
+	DBInstancedatabaseId string `json:"dbinstancedatabase_id"`
 	// 权限
-	Privileges string
+	Privileges string `json:"privileges"`
 }
 
 type DBInstanceAccountDetails struct {

pkg/apis/compute/dbinstance_const.go

@@ -16,19 +16,20 @@ package compute
 
 const (
 	//实例状态
-	DBINSTANCE_DEPLOYING      = "deploying"  //部署中
-	DBINSTANCE_RUNNING        = "running"    //运行中
-	DBINSTANCE_REBOOTING      = "rebooting"  //重启中
-	DBINSTANCE_MIGRATING      = "migrating"  //迁移中
-	DBINSTANCE_BACKING_UP     = "backing_up" //备份中
-	DBINSTANCE_RESTORING      = "restoring"  //备份恢复中
-	DBINSTANCE_RESTORE_FAILED = "restore_failed"
-	DBINSTANCE_IMPORTING      = "importing"     //数据导入中
-	DBINSTANCE_CLONING        = "cloning"       //克隆中
-	DBINSTANCE_DELETING       = "deleting"      //删除中
-	DBINSTANCE_DELETE_FAILED  = "delete_failed" //删除失败
-	DBINSTANCE_MAINTENANCE    = "maintenance"   //维护中
-	DBINSTANCE_UNKNOWN        = "unknown"
+	DBINSTANCE_DEPLOYING         = "deploying"         //部署中
+	DBINSTANCE_RUNNING           = "running"           //运行中
+	DBINSTANCE_REBOOTING         = "rebooting"         //重启中
+	DBINSTANCE_MIGRATING         = "migrating"         //迁移中
+	DBINSTANCE_BACKING_UP        = "backing_up"        //备份中
+	DBINSTANCE_BACKING_UP_FAILED = "backing_up_failed" //备份失败
+	DBINSTANCE_RESTORING         = "restoring"         //备份恢复中
+	DBINSTANCE_RESTORE_FAILED    = "restore_failed"
+	DBINSTANCE_IMPORTING         = "importing"     //数据导入中
+	DBINSTANCE_CLONING           = "cloning"       //克隆中
+	DBINSTANCE_DELETING          = "deleting"      //删除中
+	DBINSTANCE_DELETE_FAILED     = "delete_failed" //删除失败
+	DBINSTANCE_MAINTENANCE       = "maintenance"   //维护中
+	DBINSTANCE_UNKNOWN           = "unknown"
 
 	DBINSTANCE_CHANGE_CONFIG        = "change_config"        //调整配置
 	DBINSTANCE_CHANGE_CONFIG_FAILED = "change_config_failed" //调整配置失败
@@ -103,6 +104,10 @@ const (
 	HUAWEI_DBINSTANCE_CATEGORY_SINGLE  = "single"  //单机
 	HUAWEI_DBINSTANCE_CATEGORY_REPLICA = "replica" //只读
 
+	//谷歌云实例类型
+	GOOGLE_DBINSTANCE_CATEGORY_REGIONAL = "Regional" // 高可用性（区域级）
+	GOOGLE_DBINSTANCE_CATEGORY_ZONAL    = "Zonal"    // 单个地区
+
 	//阿里云存储类型
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD  = "local_ssd"  //本地盘SSD盘
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_CLOUD_ESSD = "cloud_essd" //ESSD云盘

pkg/apis/compute/externalproject.go

@@ -16,9 +16,16 @@ package compute
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const (
+	EXTERNAL_PROJECT_STATUS_AVAILABLE   = "available"   // 可用
+	EXTERNAL_PROJECT_STATUS_UNAVAILABLE = "unavailable" // 不可用
+	EXTERNAL_PROJECT_STATUS_CREATING    = "creating"    // 创建中
+	EXTERNAL_PROJECT_STATUS_DELETING    = "deleting"    // 删除中
+	EXTERNAL_PROJECT_STATUS_UNKNOWN     = "unknown"     // 未知
+)
+
 type ExternalProjectDetails struct {
-	apis.StandaloneResourceDetails
-	apis.ProjectizedResourceInfo
+	apis.VirtualResourceDetails
 	ManagedResourceInfo
 
 	SExternalProject

pkg/apis/compute/geo_input.go

@@ -64,6 +64,23 @@ type ZonalFilterListBase struct {
 	OrderByZone string `json:"order_by_zone"`
 }
 
+func (input ZonalFilterListBase) ZoneList() []string {
+	zoneStr := input.Zone
+	if len(zoneStr) > 0 {
+		input.Zones = append(input.Zones, zoneStr)
+	}
+	return input.Zones
+}
+
+func (input ZonalFilterListBase) FirstZone() string {
+	if len(input.Zone) > 0 {
+		return input.Zone
+	}
+	if len(input.Zones) > 0 {
+		return input.Zones[0]
+	}
+	return ""
+}
 func (input ZonalFilterListInput) ZoneList() []string {
 	zoneStr := input.Zone
 	if len(zoneStr) > 0 {

pkg/apis/compute/guests.go

@@ -71,8 +71,8 @@ type ServerListInput struct {
 	// 按主机资源类型进行排序
 	// enum: shared,prepaid,dedicated
 	ResourceType string `json:"resource_type"`
-	// 返回开启主备机功能的主机
-	GetBackupGuestsOnHost *bool `json:"get_backup_guests_on_host"`
+	// 返回该宿主机上的所有虚拟机，包括备份机
+	GetAllGuestsOnHost string `json:"get_all_guests_on_host"`
 
 	// 根据宿主机 SN 过滤
 	// HostSn string `json:"host_sn"`

pkg/apis/compute/guesttemplate.go

@@ -57,6 +57,7 @@ type GuestTemplateDetails struct {
 
 	Secgroups []string `json:"secgroups"`
 	Zone      string   `json:"zone"`
+	ZoneId    string   `json:"zone_id"`
 	Brand     string   `json:"brand"`
 
 	ConfigInfo GuestTemplateConfigInfo `json:"config_info"`
@@ -71,9 +72,7 @@ type GuestTemplateListInput struct {
 }
 
 type GuestTemplateConfigInfo struct {
-	Region               string                 `json:"region"`
-	Zone                 string                 `json:"zone"`
-	Hypervisor           string                 `json:"hypervisor"`
+	Metadata             map[string]string      `json:"metadata"`
 	Secgroup             string                 `json:"secgroup"`
 	Sku                  GuestTemplateSku       `json:"sku"`
 	Disks                []GuestTemplateDisk    `json:"disks"`

pkg/apis/compute/host.go

@@ -138,7 +138,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率
@@ -164,7 +164,8 @@ type HostDetails struct {
 	CanPrepare        bool                `json:"can_prepare"`
 	PrepareFailReason string              `json:"prepare_fail_reason"`
 	// 允许开启宿主机健康检查
-	AllowHealthCheck bool `json:"allow_health_check"`
+	AllowHealthCheck      bool `json:"allow_health_check"`
+	AutoMigrateOnHostDown bool `json:"auto_migrate_on_host_down"`
 
 	// 标签
 	Metadata map[string]string `json:"metadata"`

pkg/apis/compute/input.go

@@ -66,8 +66,7 @@ type CachedimageListInput struct {
 }
 
 type ExternalProjectListInput struct {
-	apis.StandaloneResourceListInput
-	apis.ProjectizedResourceListInput
+	apis.VirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput

pkg/apis/compute/loadbalancer.go

@@ -48,6 +48,8 @@ type LoadbalancerListenerListInput struct {
 
 	Scheduler []string `json:"scheduler"`
 
+	Certificate []string `json:"certificate_id"`
+
 	SendProxy []string `json:"send_proxy"`
 
 	AclStatus []string `json:"acl_status"`
@@ -168,6 +170,8 @@ type LoadbalancerDetails struct {
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
+	LoadbalancerClusterResourceInfo
+
 	VpcResourceInfoBase
 	ZoneResourceInfoBase
 	NetworkResourceInfoBase

pkg/apis/compute/scaling_group.go

@@ -134,9 +134,15 @@ type ScalingGroupDetails struct {
 	// example: OneCloud
 	Brand string `json:"brand"`
 
-	// description: 网络ID
-	// example: net-12345
-	Networks []string `json:"networks"`
+	// description: 网络信息
+	Networks []ScalingGroupNetwork `json:"networks"`
+}
+
+type ScalingGroupNetwork struct {
+	Id           string `json:"id"`
+	Name         string `json:"name"`
+	GuestIpStart string `json:"guest_ip_start"`
+	GuestIpEnd   string `json:"guest_ip_end"`
 }
 
 type ScalingGroupResourceInfo struct {

pkg/apis/compute/secgroup.go

@@ -145,8 +145,18 @@ type SecgroupListInput struct {
 	// pattern:asc|desc
 	OrderByGuestCnt string `json:"order_by_guest_cnt"`
 
-	// 是否被修改
-	IsDirty *bool `json:"is_dirty"`
+	// 模糊过滤规则中含有指定ip的安全组
+	// example: 10.10.2.1
+	Ip string `json:"ip"`
+
+	// 精确匹配规则中含有指定端口的安全组
+	// example: 100-200
+	Ports string `json:"ports"`
+
+	// 指定过滤规则的方向(仅在指定ip或ports时生效) choices: all|in|out
+	// default: all
+	// example: in
+	Direction string `json:"direction"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -164,12 +174,18 @@ type SecurityGroupRuleListInput struct {
 	apis.ResourceBaseListInput
 	SecgroupFilterListInput
 
+	Projects []string `json:"projects"`
+
 	// 以direction字段过滤安全组规则
 	Direction string `json:"direction"`
 	// 以action字段过滤安全组规则
 	Action string `json:"action"`
 	// 以protocol字段过滤安全组规则
 	Protocol string `json:"protocol"`
+	// 以ports字段过滤安全组规则
+	Ports string `json:"ports"`
+	// 根据ip模糊匹配安全组规则
+	Ip string `json:"ip"`
 }
 
 type SecgroupResourceInput struct {
@@ -179,6 +195,9 @@ type SecgroupResourceInput struct {
 	// Deprecated
 	// filter by secgroup_id
 	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
+
+	// 模糊匹配安全组规则名称
+	SecgroupName string `json:"secgroup_name"`
 }
 
 type SecgroupFilterListInput struct {
@@ -193,15 +212,22 @@ type SecgroupDetails struct {
 	SSecurityGroup
 
 	// 关联云主机数量
-	GuestCnt int `json:"guest_cnt"`
+	GuestCnt int `json:"guest_cnt,allowempty"`
+
+	// 关联此安全组的云主机is_system为true数量
+	SystemGuestCnt int `json:"system_guest_cnt,allowempty"`
+
+	// admin_secgrp_id为此安全组的云主机数量
+	AdminGuestCnt int `json:"admin_guest_cnt,allowempty"`
+
 	// 安全组缓存数量
-	CacheCnt int `json:"cache_cnt"`
+	CacheCnt int `json:"cache_cnt,allowempty"`
 	// 规则信息
-	Rules string `json:"rules"`
+	Rules []SSecurityGroupRule `json:"rules"`
 	// 入方向规则信息
-	InRules string `json:"in_rules"`
+	InRules []SSecurityGroupRule `json:"in_rules"`
 	// 出方向规则信息
-	OutRules string `json:"out_rules"`
+	OutRules []SSecurityGroupRule `json:"out_rules"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/storage_const.go

@@ -127,7 +127,7 @@ var (
 		STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS,
 	}
 
-	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA}
 
 	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
 

pkg/apis/identity/config.go

@@ -28,6 +28,8 @@ type SLDAPIdpConfigBaseOptions struct {
 	Suffix   string `json:"suffix,omitempty" required:"true"`
 	User     string `json:"user,omitempty" required:"true"`
 	Password string `json:"password,omitempty" required:"true"`
+
+	DisableUserOnImport bool `json:"disable_user_on_import"`
 }
 
 type SLDAPIdpConfigSingleDomainOptions struct {
@@ -51,6 +53,8 @@ type SLDAPIdpConfigOptions struct {
 	User     string `json:"user,omitempty"`
 	Password string `json:"password,omitempty"`
 
+	DisableUserOnImport bool `json:"disable_user_on_import"`
+
 	DomainTreeDN        string `json:"domain_tree_dn,omitempty" help:"Domain tree root node dn(distinguished name)"`
 	DomainFilter        string `json:"domain_filter,omitempty"`
 	DomainObjectclass   string `json:"domain_objectclass,omitempty"`

pkg/apis/identity/input.go

@@ -151,6 +151,9 @@ type ProjectListInput struct {
 
 	UserFilterListInput
 	GroupFilterListInput
+
+	// 过滤出指定用户或者组可以加入的项目
+	Jointable *bool `json:"jointable"`
 }
 
 type DomainListInput struct {

pkg/appsrv/ring.go

@@ -25,10 +25,12 @@ type Ring struct {
 }
 
 func NewRing(size int) *Ring {
-	r := Ring{buffer: make([]interface{}, size+1),
+	r := Ring{
+		buffer: make([]interface{}, size+1),
 		header: 0,
 		tail:   0,
-		lock:   &sync.Mutex{}}
+		lock:   &sync.Mutex{},
+	}
 	return &r
 }
 
@@ -58,6 +60,7 @@ func (r *Ring) Pop() interface{} {
 		return nil
 	}
 	ret := r.buffer[r.tail]
+	r.buffer[r.tail] = nil
 	r.tail = nextPointer(r.tail, len(r.buffer))
 	return ret
 }

pkg/appsrv/ring_test.go

@@ -19,28 +19,33 @@ import (
 )
 
 func TestRing(t *testing.T) {
-	r := NewRing(10)
-	var v int32 = 10
-	r.Push(v)
-	v = 20
-	r.Push(v)
-	v = 30
-	r.Push(v)
-	v1 := r.Pop().(int32)
-	if v1 != 10 {
-		t.Error("Fail")
-	}
-	v2 := r.Pop().(int32)
-	if v2 != 20 {
-		t.Error("Fail")
-	}
-	v3 := r.Pop().(int32)
-	if v3 != 30 {
-		t.Error("Fail")
-	}
-	v4 := r.Pop()
-	if v4 != nil {
-		t.Error("Fail")
+	var (
+		r    = NewRing(10)
+		push = func(v int32) {
+			r.Push(v)
+		}
+		pop = func(want int32) {
+			got := r.Pop().(int32)
+			if got != want {
+				t.Fatalf("got %d, want %d", got, want)
+			}
+			for i := r.header; i != r.tail; i = nextPointer(i, len(r.buffer)) {
+				if r.buffer[i] != nil {
+					t.Fatalf("head %d, tail %d, index %d not nil",
+						r.header, r.tail, i)
+				}
+			}
+		}
+	)
+	push(10)
+	push(20)
+	push(30)
+
+	pop(10)
+	pop(20)
+	pop(30)
+	if v := r.Pop(); v != nil {
+		t.Fatalf("want nil, got %#v", v)
 	}
 }
 

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/app/auth.go

@@ -87,8 +87,9 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
-		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
-			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,
+		policy.EnableGlobalRbac(
+			time.Second*time.Duration(options.RbacPolicySyncPeriodSeconds),
+			time.Second*time.Duration(options.RbacPolicySyncFailedRetrySeconds),
 			options.RbacDebug,
 		)
 	}

pkg/cloudcommon/db/domain.go

@@ -77,6 +77,10 @@ func (model *SDomainizedResourceBase) GetChangeOwnerCandidateDomainIds() []strin
 	return nil
 }
 
+func (model *SDomainizedResourceBase) GetChangeOwnerRequiredDomainIds() []string {
+	return nil
+}
+
 func ValidateCreateDomainId(domainId string) error {
 	if !consts.GetNonDefaultDomainProjects() && domainId != identity.DEFAULT_DOMAIN_ID {
 		return httperrors.NewForbiddenError("project in non-default domain is prohibited")

pkg/cloudcommon/db/domainresource.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -133,6 +134,9 @@ func (model *SDomainLevelResourceBase) AllowPerformChangeOwner(ctx context.Conte
 }
 
 func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformChangeDomainOwnerInput) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.GetIStandaloneModel().IsShared() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot change owner of shared resource")
 	}
@@ -162,6 +166,11 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 	if len(candidates) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), candidates) {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner candidate list")
 	}
+	requires := model.GetIDomainLevelModel().GetChangeOwnerRequiredDomainIds()
+	log.Debugf("%s required domains: %s", model.Keyword(), requires)
+	if len(requires) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), requires) {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner required list")
+	}
 
 	if !IsAdminAllowPerform(userCred, model, "change-owner") {
 		return nil, errors.Wrap(httperrors.ErrNotSufficientPrivilege, "require system privileges")

pkg/cloudcommon/db/external.go

@@ -66,7 +66,14 @@ func SetExternalId(model IExternalizedModel, userCred mcclient.TokenCredential,
 }
 
 func FetchByExternalId(manager IModelManager, idStr string) (IExternalizedModel, error) {
+	return FetchByExternalIdAndManagerId(manager, idStr, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		return q
+	})
+}
+
+func FetchByExternalIdAndManagerId(manager IModelManager, idStr string, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (IExternalizedModel, error) {
 	q := manager.Query().Equals("external_id", idStr)
+	q = filter(q)
 	count, err := q.CountWithError()
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/infraresource.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -200,6 +201,9 @@ func (model *SInfrasResourceBase) PerformChangeOwner(
 	query jsonutils.JSONObject,
 	input apis.PerformChangeDomainOwnerInput,
 ) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.IsShared() {
 		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot change owner when shared!")
 	}

pkg/cloudcommon/db/lockman/base.go

@@ -0,0 +1,51 @@
+package lockman
+
+import "context"
+
+type SBaseLockManager struct {
+	manager ILockManager
+}
+
+func NewBaseLockManger(m ILockManager) *SBaseLockManager {
+	return &SBaseLockManager{manager: m}
+}
+
+func (m *SBaseLockManager) LockClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.UnlockKey(ctx, key)
+}

pkg/cloudcommon/db/lockman/etcd.go

@@ -167,6 +167,7 @@ type SLockTableIndex struct {
 }
 
 type SEtcdLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[SLockTableIndex]*SEtcdLockRecord
 
@@ -203,6 +204,7 @@ func NewEtcdLockManager(config *SEtcdLockManagerConfig) (ILockManager, error) {
 		Value:  lockman,
 		Func:   atexit.ExitHandlerFunc(lockman.destroyAtExit),
 	})
+	lockman.SBaseLockManager = NewBaseLockManger(&lockman)
 	return &lockman, nil
 }
 

pkg/cloudcommon/db/lockman/inmemory.go

@@ -135,12 +135,17 @@ func (rec *SInMemoryLockRecord) unlockContext(ctx context.Context) (needClean bo
 }
 
 type SInMemoryLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[string]*SInMemoryLockRecord
 }
 
 func NewInMemoryLockManager() ILockManager {
-	lockMan := SInMemoryLockManager{tableLock: &sync.Mutex{}, lockTable: make(map[string]*SInMemoryLockRecord)}
+	lockMan := SInMemoryLockManager{
+		tableLock: &sync.Mutex{},
+		lockTable: make(map[string]*SInMemoryLockRecord),
+	}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }
 

pkg/cloudcommon/db/lockman/lockman.go

@@ -31,6 +31,15 @@ type ILockedObject interface {
 type ILockManager interface {
 	LockKey(ctx context.Context, key string)
 	UnlockKey(ctx context.Context, key string)
+
+	LockClass(ctx context.Context, manager ILockedClass, projectId string)
+	ReleaseClass(ctx context.Context, manager ILockedClass, projectId string)
+	LockObject(ctx context.Context, model ILockedObject)
+	ReleaseObject(ctx context.Context, model ILockedObject)
+	LockRawObject(ctx context.Context, resName string, resId string)
+	ReleaseRawObject(ctx context.Context, resName string, resId string)
+	LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
+	ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
 }
 
 func getClassKey(manager ILockedClass, projectId string) string {
@@ -64,41 +73,33 @@ func Init(man ILockManager) {
 }
 
 func LockClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockClass(ctx, manager, projectId)
 }
 
 func ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseClass(ctx, manager, projectId)
 }
 
 func LockObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockObject(ctx, model)
 }
 
 func ReleaseObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseObject(ctx, model)
 }
 
 func LockRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockRawObject(ctx, resName, resId)
 }
 
 func ReleaseRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseRawObject(ctx, resName, resId)
 }
 
 func LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockJointObject(ctx, model, model2)
 }
 
 func ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseJointObject(ctx, model, model2)
 }

pkg/cloudcommon/db/lockman/noop.go

@@ -21,6 +21,7 @@ import (
 )
 
 type SNoopLockManager struct {
+	*SBaseLockManager
 }
 
 func (lockman *SNoopLockManager) LockKey(ctx context.Context, key string) {
@@ -33,5 +34,6 @@ func (lockman *SNoopLockManager) UnlockKey(ctx context.Context, key string) {
 
 func NewNoopLockManager() ILockManager {
 	lockMan := SNoopLockManager{}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }

pkg/cloudcommon/db/managed.go

@@ -22,6 +22,7 @@ import (
 
 type IOwnerResourceBaseModel interface {
 	GetChangeOwnerCandidateDomainIds() []string
+	GetChangeOwnerRequiredDomainIds() []string
 }
 
 type IManagedResourceBase interface {

pkg/cloudcommon/db/opslog.go

@@ -621,17 +621,11 @@ func (self *SOpsLogManager) FilterByOwner(q *sqlchemy.SQuery, ownerId mcclient.I
 			}
 		case rbacutils.ScopeProject:
 			if len(ownerId.GetProjectId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_tenant_id"), ownerId.GetProjectId()),
-					sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()))
 			}
 		case rbacutils.ScopeDomain:
 			if len(ownerId.GetProjectDomainId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_domain_id"), ownerId.GetProjectDomainId()),
-					sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()))
 			}
 		default:
 			// systemScope, no filter

pkg/cloudcommon/db/quotas/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -481,7 +482,7 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) {
@@ -524,7 +525,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
@@ -616,3 +617,33 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 	}
 	return ret, nil
 }
+
+type tQuotaResultList []jsonutils.JSONObject
+
+func (a tQuotaResultList) Len() int           { return len(a) }
+func (a tQuotaResultList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a tQuotaResultList) Less(i, j int) bool { return usageRateOfQuota(a[i]) > usageRateOfQuota(a[j]) }
+
+func usageRateOfQuota(quota jsonutils.JSONObject) float32 {
+	maxRate := float32(0)
+	quotaMap, _ := quota.GetMap()
+	for k, v := range quotaMap {
+		usageK := fmt.Sprintf("usage.%s", k)
+		if usageV, ok := quotaMap[usageK]; ok {
+			intV, _ := v.Int()
+			if intV > 0 {
+				intUsageV, _ := usageV.Int()
+				rate := float32(intUsageV) / float32(intV)
+				if maxRate < rate {
+					maxRate = rate
+				}
+			}
+		}
+	}
+	return maxRate
+}
+
+func sortQuotaByUsage(quotaList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	sort.Sort(tQuotaResultList(quotaList))
+	return quotaList
+}

pkg/cloudcommon/db/quotas/interface.go

@@ -71,6 +71,7 @@ type IQuotaManager interface {
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
 	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)

pkg/cloudcommon/db/quotas/quotas.go

@@ -113,6 +113,17 @@ func (manager *SQuotaBaseManager) _cancelUsage(ctx context.Context, userCred mcc
 	return manager.changeUsage(ctx, userCred, usage, false)
 }
 
+func (manager *SQuotaBaseManager) addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	LockQuota(ctx, manager, usage)
+	defer ReleaseQuota(ctx, manager, usage)
+
+	return manager._addUsage(ctx, userCred, usage)
+}
+
+func (manager *SQuotaBaseManager) _addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	return manager.changeUsage(ctx, userCred, usage, true)
+}
+
 func (manager *SQuotaBaseManager) changeUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota, isAdd bool) error {
 	usages, err := manager.usageStore.GetParentQuotas(ctx, usage.GetKeys())
 	if err != nil {

pkg/cloudcommon/db/quotas/register.go

@@ -87,6 +87,20 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 	}
 }
 
+func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	for _, usage := range usages {
+		addUsage(ctx, userCred, usage.(IQuota))
+	}
+}
+
+func addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) {
+	manager := getQuotaManager(usage)
+	err := manager.addUsage(ctx, userCred, usage)
+	if err != nil {
+		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
+	}
+}
+
 func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
 	manager := getQuotaManager(request)
 	return manager.getQuotaCount(ctx, request, pendingKeys)

pkg/cloudcommon/db/sharablebase.go

@@ -162,8 +162,14 @@ func SharableManagerValidateCreateData(
 			input.IsPublic = &isPublic
 			reqScope = rbacutils.ScopeSystem
 		} else if input.PublicScope == string(rbacutils.ScopeDomain) {
-			input.IsPublic = &isPublic
-			reqScope = rbacutils.ScopeDomain
+			if consts.GetNonDefaultDomainProjects() {
+				// only if non_default_domain_projects turned on, allow sharing to domain
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeDomain
+			} else {
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeSystem
+			}
 		} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
 			// backward compatible, if only is_public is true, make it share to system
 			input.IsPublic = &isPublic
@@ -171,17 +177,23 @@ func SharableManagerValidateCreateData(
 			reqScope = rbacutils.ScopeSystem
 		} else {
 			input.IsPublic = nil
-			input.PublicScope = string(rbacutils.ScopeNone)
+			input.PublicScope = "" // string(rbacutils.ScopeNone)
 		}
 	case rbacutils.ScopeDomain:
-		if input.PublicScope == string(rbacutils.ScopeSystem) {
-			input.IsPublic = &isPublic
-			reqScope = rbacutils.ScopeSystem
-		} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
-			// backward compatible, if only is_public is true, make it share to system
-			input.IsPublic = &isPublic
-			input.PublicScope = string(rbacutils.ScopeSystem)
-			reqScope = rbacutils.ScopeSystem
+		if consts.GetNonDefaultDomainProjects() {
+			// only if non_default_domain_projects turned on, allow sharing domain resources
+			if input.PublicScope == string(rbacutils.ScopeSystem) {
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeSystem
+			} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
+				// backward compatible, if only is_public is true, make it share to system
+				input.IsPublic = &isPublic
+				input.PublicScope = string(rbacutils.ScopeSystem)
+				reqScope = rbacutils.ScopeSystem
+			} else {
+				input.IsPublic = nil
+				input.PublicScope = "" // string(rbacutils.ScopeNone)
+			}
 		} else {
 			input.IsPublic = nil
 			input.PublicScope = string(rbacutils.ScopeNone)
@@ -423,6 +435,9 @@ func SharablePerformPublic(model ISharableBaseModel, ctx context.Context, userCr
 			targetScope = rbacutils.ScopeNone
 		}
 	case rbacutils.ScopeDomain:
+		if !consts.GetNonDefaultDomainProjects() {
+			return errors.Wrap(httperrors.ErrForbidden, "not allow to share to domain when non_default_domain_projects turned off")
+		}
 		if len(requireIds) == 0 {
 			return errors.Wrap(httperrors.ErrForbidden, "require to be shared to system")
 		}
@@ -555,7 +570,8 @@ func SharableModelIsShared(model ISharableBaseModel) bool {
 func SharableModelCustomizeCreate(model ISharableBaseModel, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	if !data.Contains("public_scope") {
 		resScope := model.GetModelManager().ResourceScope()
-		if resScope == rbacutils.ScopeDomain {
+		if resScope == rbacutils.ScopeDomain && consts.GetNonDefaultDomainProjects() {
+			// only if non_default_domain_projects turned on, do the following
 			isManaged := false
 			if managedModel, ok := model.(IManagedResourceBase); ok {
 				isManaged = managedModel.IsManaged()

pkg/cloudcommon/db/standalone.go

@@ -488,6 +488,10 @@ func (manager *SStandaloneResourceBaseManager) ValidateCreateData(ctx context.Co
 	if err != nil {
 		return input, errors.Wrap(err, "SResourceBaseManager.ValidateCreateData")
 	}
+	input.Name = strings.TrimSpace(input.Name)
+	if strings.ContainsAny(input.Name, "\n\r\t") {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "name should not contains any \\n\\r\\t")
+	}
 	return input, nil
 }
 
@@ -578,6 +582,10 @@ func (model *SStandaloneResourceBase) ValidateUpdateData(ctx context.Context, us
 	}
 
 	if len(input.Name) > 0 {
+		input.Name = strings.TrimSpace(input.Name)
+		if strings.ContainsAny(input.Name, "\n\r\t") {
+			return input, errors.Wrap(httperrors.ErrInputParameter, "name should not contains any \\n\\r\\t")
+		}
 		err = alterNameValidator(model.GetIStandaloneModel(), input.Name)
 		if err != nil {
 			return input, errors.Wrap(err, "alterNameValidator")

pkg/cloudcommon/policy/defaults.go

@@ -29,6 +29,11 @@ var (
 					Action:   PolicyActionPerform,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Resource: "metadatas",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 		{

pkg/cloudcommon/policy/policy.go

@@ -20,6 +20,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -186,16 +187,38 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 			policiesMap[policy.Scope] = policies
 		}
 		manager.defaultPolicies = policiesMap
-		log.Debugf("%#v", manager.defaultPolicies)
+		// log.Debugf("%#v", manager.defaultPolicies)
 	}
 
 	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
 
-	manager.SyncOnce()
+	manager.syncByInterval()
 }
 
+func (manager *SPolicyManager) syncByInterval() {
+	syncWorkerManager.Run(manager.syncByInterval_, nil, nil)
+}
+
+func (manager *SPolicyManager) syncByInterval_() {
+	err := manager.doSync()
+	var interval time.Duration
+	if err != nil {
+		interval = manager.failedRetryInterval
+	} else {
+		interval = manager.refreshInterval
+	}
+	time.AfterFunc(interval, manager.syncByInterval)
+}
+
+var syncOnce int32
+
 func (manager *SPolicyManager) SyncOnce() {
-	syncWorkerManager.Run(manager.sync, nil, nil)
+	if atomic.CompareAndSwapInt32(&syncOnce, 0, 1) {
+		syncWorkerManager.Run(func() {
+			atomic.StoreInt32(&syncOnce, 0)
+			manager.doSync()
+		}, nil, nil)
+	}
 }
 
 func (manager *SPolicyManager) doSync() error {
@@ -223,17 +246,6 @@ func (manager *SPolicyManager) doSync() error {
 	return nil
 }
 
-func (manager *SPolicyManager) sync() {
-	err := manager.doSync()
-	var interval time.Duration
-	if err != nil {
-		interval = manager.failedRetryInterval
-	} else {
-		interval = manager.refreshInterval
-	}
-	time.AfterFunc(interval, manager.SyncOnce)
-}
-
 func queryKey(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) string {
 	queryKeys := []string{string(scope)}
 	queryKeys = append(queryKeys, userCred.GetProjectId(), userCred.GetDomainId(), userCred.GetUserId())
@@ -403,8 +415,15 @@ func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, use
 
 	var result rbacutils.TRbacResult
 	if len(matchRules) > 0 {
-		rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
-		result = rule.Result
+		result = rbacutils.Deny
+		for _, rule := range matchRules {
+			if rule.Result == rbacutils.Allow {
+				result = rbacutils.Allow
+				break
+			}
+		}
+		// rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
+		// result = rule.Result
 	} else if findMatchPolicy {
 		// if find matched policy, but no rule matching, allow anyway
 		result = rbacutils.Allow

pkg/cloudprovider/cloudprovider.go

@@ -190,6 +190,8 @@ type ICloudProviderFactory interface {
 	IsCloudeventRegional() bool
 	GetMaxCloudEventSyncDays() int
 	GetMaxCloudEventKeepDays() int
+
+	IsNeedForceAutoCreateProject() bool
 }
 
 type ICloudProvider interface {
@@ -215,6 +217,8 @@ type ICloudProvider interface {
 	GetCloudRegionExternalIdPrefix() string
 
 	GetStorageClasses(regionId string) []string
+	GetBucketCannedAcls(regionId string) []string
+	GetObjectCannedAcls(regionId string) []string
 
 	GetCapabilities() []string
 	GetICloudQuotas() ([]ICloudQuota, error)
@@ -408,6 +412,10 @@ func (factory *baseProviderFactory) GetMaxCloudEventKeepDays() int {
 	return 7
 }
 
+func (factory *baseProviderFactory) IsNeedForceAutoCreateProject() bool {
+	return false
+}
+
 type SPremiseBaseProviderFactory struct {
 	baseProviderFactory
 }

pkg/cloudprovider/dbinstance.go

@@ -88,6 +88,7 @@ type SDBInstanceBackupCreateConfig struct {
 }
 
 type SDBInstanceRecoveryConfig struct {
-	BackupId  string
-	Databases map[string]string
+	BackupId                   string
+	Databases                  map[string]string
+	OriginDBInstanceExternalId string
 }

pkg/cloudprovider/fakeregion.go

@@ -97,7 +97,7 @@ func (region *SFakeOnPremiseRegion) GetISecurityGroupById(id string) (ICloudSecu
 	return nil, ErrNotSupported
 }
 
-func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error) {
+func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error) {
 	return nil, ErrNotSupported
 }
 

pkg/cloudprovider/images.go

@@ -49,6 +49,7 @@ type SImage struct {
 	Status     string
 	// UpdatedAt       time.Time
 	PublicScope string
+	ExternalId  string
 }
 
 func CloudImage2Image(image ICloudImage) SImage {

pkg/cloudprovider/network.go

@@ -0,0 +1,22 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudprovider
+
+type SNetworkCreateOptions struct {
+	Name      string
+	Desc      string
+	ProjectId string
+	Cidr      string
+}

pkg/cloudprovider/resources.go

@@ -20,10 +20,10 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/tristate"
-	"yunion.io/x/pkg/util/secrules"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type ICloudResource interface {
@@ -71,7 +71,7 @@ type ICloudRegion interface {
 	GetIDiskById(id string) (ICloudDisk, error)
 
 	GetISecurityGroupById(secgroupId string) (ICloudSecurityGroup, error)
-	GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error)
+	GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error)
 	CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error)
 
 	CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error)
@@ -343,13 +343,13 @@ type ICloudEIP interface {
 }
 
 type ICloudSecurityGroup interface {
-	ICloudResource
+	IVirtualResource
 
 	GetDescription() string
-	GetRules() ([]secrules.SecurityRule, error)
+	GetRules() ([]SecurityRule, error)
 	GetVpcId() string
 
-	SyncRules(rules []secrules.SecurityRule) error
+	SyncRules(common, inAdds, outAdds, inDels, outDels []SecurityRule) error
 	Delete() error
 }
 
@@ -451,7 +451,7 @@ type ICloudWire interface {
 
 	GetINetworkById(netid string) (ICloudNetwork, error)
 
-	CreateINetwork(name string, cidr string, desc string) (ICloudNetwork, error)
+	CreateINetwork(opts *SNetworkCreateOptions) (ICloudNetwork, error)
 }
 
 type ICloudNetwork interface {
@@ -464,8 +464,11 @@ type ICloudNetwork interface {
 	GetIpMask() int8
 	GetGateway() string
 	GetServerType() string
-	// GetIsPublic() bool
-	// GetPublicScope() rbacutils.TRbacScope
+	//GetIsPublic() bool
+	// 仅私有云有用，公有云无效
+	// 1. scope = none 非共享, network仅会属于一个项目,并且私有
+	// 2. scope = system 系统共享 云账号共享会跟随云账号共享，云账号非共享,会共享到network所在域
+	GetPublicScope() rbacutils.TRbacScope
 
 	Delete() error
 

pkg/cloudprovider/securitygroup.go

@@ -14,11 +14,291 @@
 
 package cloudprovider
 
-import "yunion.io/x/pkg/util/secrules"
+import (
+	"sort"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/secrules"
+)
+
+const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
+
+type SecurityGroupFilterOptions struct {
+	VpcId     string
+	Name      string
+	ProjectId string
+}
 
 type SecurityGroupCreateInput struct {
-	Name  string
-	Desc  string
-	VpcId string
-	Rules []secrules.SecurityRule
+	Name      string
+	Desc      string
+	VpcId     string
+	ProjectId string
+	Rules     []secrules.SecurityRule
+}
+
+type SecurityRule struct {
+	secrules.SecurityRule
+	Name       string
+	ExternalId string
+}
+
+type TPriorityOrder int
+
+var (
+	PriorityOrderByDesc = TPriorityOrder(1)
+	PriorityOrderByAsc  = TPriorityOrder(-1)
+)
+
+func (r SecurityRule) String() string {
+	return r.SecurityRule.String()
+}
+
+type SecurityRuleSet []SecurityRule
+
+func (srs SecurityRuleSet) Len() int {
+	return len(srs)
+}
+
+func (srs SecurityRuleSet) Swap(i, j int) {
+	srs[i], srs[j] = srs[j], srs[i]
+}
+
+func (srs SecurityRuleSet) Less(i, j int) bool {
+	return srs[i].Priority < srs[j].Priority || (srs[i].Priority == srs[j].Priority && srs[i].String() < srs[j].String())
+}
+
+func (srs SecurityRuleSet) AllowList() secrules.SecurityRuleSet {
+	rules := secrules.SecurityRuleSet{}
+	for _, r := range srs {
+		rules = append(rules, r.SecurityRule)
+	}
+	return rules.AllowList()
+}
+
+func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr string, order TPriorityOrder, min, max int, onlyAllowRules bool) []SecurityRule {
+	if defaultRule.String() == localRuleStr {
+		return rules
+	}
+	defaultRule.ExternalId = DEFAULT_CLOUD_RULE_ID
+	if order == PriorityOrderByDesc {
+		defaultRule.Priority = min
+	} else {
+		defaultRule.Priority = max
+	}
+	defaultRule.Priority -= int(order)
+	if onlyAllowRules {
+		defaultRule.Priority = -1
+	}
+	return append(rules, defaultRule)
+}
+
+func SortSecurityRule(rules SecurityRuleSet, order TPriorityOrder, onlyAllowRules bool) {
+	if onlyAllowRules {
+		sort.Sort(rules)
+		return
+	}
+	if order == PriorityOrderByAsc {
+		sort.Sort(sort.Reverse(rules))
+		return
+	}
+	sort.Sort(rules)
+}
+
+func CompareRules(
+	minPriority, maxPriority int, order TPriorityOrder,
+	localRules secrules.SecurityRuleSet, remoteRules []SecurityRule,
+	defaultInRule, defaultOutRule SecurityRule,
+	onlyAllowRules bool, debug bool,
+) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
+	localInRules := secrules.SecurityRuleSet{}
+	localOutRules := secrules.SecurityRuleSet{}
+	for i := range localRules {
+		if localRules[i].Direction == secrules.DIR_IN {
+			localInRules = append(localInRules, localRules[i])
+		} else {
+			localOutRules = append(localOutRules, localRules[i])
+		}
+	}
+	inRules := SecurityRuleSet{}
+	outRules := SecurityRuleSet{}
+	for i := 0; i < len(remoteRules); i++ {
+		if remoteRules[i].Direction == secrules.DIR_IN {
+			inRules = append(inRules, remoteRules[i])
+		} else {
+			outRules = append(outRules, remoteRules[i])
+		}
+	}
+	var inCommon, outCommon = inRules, outRules
+
+	defaultLocalInRule := *secrules.MustParseSecurityRule("in:deny any")
+	defaultLocalOutRule := *secrules.MustParseSecurityRule("out:allow any")
+
+	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+
+	if defaultLocalInRule.String() != defaultInRule.String() {
+		localInRules = append(localInRules, defaultLocalInRule)
+	}
+	if defaultLocalOutRule.String() != defaultOutRule.String() {
+		localOutRules = append(localOutRules, defaultLocalOutRule)
+	}
+
+	sort.Sort(localInRules)
+	sort.Sort(localOutRules)
+
+	localInAllowList := localInRules.AllowList()
+	localOutAllowList := localOutRules.AllowList()
+	if onlyAllowRules {
+		localInRules = localInAllowList
+		localOutRules = localOutAllowList
+	}
+
+	SortSecurityRule(inRules, order, onlyAllowRules)
+	SortSecurityRule(outRules, order, onlyAllowRules)
+
+	inAllowList := inRules.AllowList()
+	outAllowList := outRules.AllowList()
+	inEquals, outEquals := inAllowList.Equals(localInAllowList), outAllowList.Equals(localOutAllowList)
+	if inEquals && outEquals {
+		return
+	}
+
+	// priority从小到大排列(从默认规则开始对比)
+	sort.Sort(sort.Reverse(localInRules))
+	sort.Sort(sort.Reverse(localOutRules))
+
+	sort.Sort(sort.Reverse(inRules))
+	sort.Sort(sort.Reverse(outRules))
+
+	startPriority := minPriority - 1
+	if order == PriorityOrderByAsc {
+		startPriority = maxPriority + 1
+	}
+
+	var addPriority = func(priority int, order TPriorityOrder, inc int, min, max int, onlyAllowRules bool) int {
+		if onlyAllowRules {
+			return 0
+		}
+		inc = inc * int(order) //+ int(order)
+		priority += inc
+		if priority < min {
+			return min
+		}
+		if priority > max {
+			return max
+		}
+		return priority
+	}
+
+	var getInitPriority = func(init, min, max int) int {
+		if init < min || init > max {
+			return (min + max) / 2
+		}
+		return init
+	}
+
+	var compare = func(localRules secrules.SecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
+		i, j, inc, prePriority := 0, 0, 1, 0
+		for i < len(localRules) || j < len(remoteRules) {
+			if i < len(localRules) && j < len(remoteRules) {
+				ruleStr := remoteRules[j].String()
+				localRuleStr := localRules[i].String()
+				if debug {
+					log.Debugf("compare local priority(%d) %s -> remote name(%s) priority(%d) %s\n", localRules[i].Priority, localRules[i].String(), remoteRules[j].Name, remoteRules[j].Priority, remoteRules[j].String())
+				}
+				cmp := strings.Compare(ruleStr, localRuleStr)
+				if cmp == 0 {
+					prePriority = remoteRules[j].Priority
+					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
+						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
+					}
+					common = append(common, remoteRules[j])
+					i++
+					j++
+				} else if cmp < 0 {
+					if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+						del = append(del, remoteRules[j])
+					}
+					j++
+				} else {
+					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
+					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+					add = append(add, SecurityRule{SecurityRule: localRules[i]})
+					i++
+					inc++
+				}
+			} else if i >= len(localRules) {
+				if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+					del = append(del, remoteRules[j])
+				}
+				j++
+			} else if j >= len(remoteRules) {
+				initPriority := startPriority
+				if len(remoteRules) > 0 {
+					initPriority = remoteRules[len(remoteRules)-1].Priority
+				}
+				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
+				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+				add = append(add, SecurityRule{SecurityRule: localRules[i]})
+				i++
+				inc++
+			}
+		}
+		return
+	}
+
+	type rulePair struct {
+		localRules  []secrules.SecurityRule
+		remoteRules []SecurityRule
+		protocol    string
+	}
+
+	var splitRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) []rulePair {
+		rules := map[string]rulePair{}
+		for _, r := range localRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.localRules = append(pair.localRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		for _, r := range remoteRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.remoteRules = append(pair.remoteRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		ret := []rulePair{}
+		for _, r := range rules {
+			ret = append(ret, r)
+		}
+		return ret
+	}
+
+	var compareRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
+		pairs := splitRules(localRules, remoteRules)
+		for _, r := range pairs {
+			_common, _add, _dels := compare(r.localRules, r.remoteRules)
+			common = append(common, _common...)
+			add = append(add, _add...)
+			dels = append(dels, _dels...)
+		}
+		return
+	}
+
+	if !inEquals {
+		inCommon, inAdds, inDels = compareRules(localInRules, inRules)
+	}
+	if !outEquals {
+		outCommon, outAdds, outDels = compareRules(localOutRules, outRules)
+	}
+	common = append(inCommon, outCommon...)
+	return
 }

pkg/compute/guestdrivers/azure.go

@@ -124,6 +124,30 @@ func (self *SAzureGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *mo
 	return nil
 }
 
+func (self *SAzureGuestDriver) ValidateImage(ctx context.Context, image *cloudprovider.SImage) error {
+	if len(image.ExternalId) == 0 {
+		s := auth.GetAdminSession(ctx, options.Options.Region, "")
+		result, err := modules.Images.GetSpecific(s, image.Id, "subformats", nil)
+		if err != nil {
+			return errors.Wrap(err, "get subformats")
+		}
+		subFormats := []struct {
+			Format string
+		}{}
+		err = result.Unmarshal(&subFormats)
+		if err != nil {
+			return errors.Wrap(err, "Unmarshal subformats")
+		}
+		for i := range subFormats {
+			if subFormats[i].Format == "vhd" {
+				return nil
+			}
+		}
+		return httperrors.NewResourceNotFoundError("failed to find subformat vhd for image %s, please append 'vhd' for glance options(target_image_formats)", image.Name)
+	}
+	return nil
+}
+
 func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) {
 	input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input)
 	if err != nil {
@@ -137,59 +161,45 @@ func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		if err != nil {
 			return nil, errors.Wrap(err, "FetchById")
 		}
-		image := _image.(*models.SCachedimage)
-		if len(image.ExternalId) == 0 {
-			s := auth.GetAdminSession(ctx, options.Options.Region, "")
-			result, err := modules.Images.GetSpecific(s, image.Id, "subformats", nil)
-			if err != nil {
-				return nil, errors.Wrap(err, "get subformats")
-			}
-			subFormats := []struct {
-				Format string
-			}{}
-			err = result.Unmarshal(&subFormats)
-			if err != nil {
-				return nil, errors.Wrap(err, "Unmarshal subformats")
-			}
-			find := false
-			for i := range subFormats {
-				if subFormats[i].Format == "vhd" {
-					find = true
-					break
+
+		cachedimage := _image.(*models.SCachedimage)
+		image, err := cachedimage.GetImage()
+		if err != nil {
+			return nil, errors.Wrap(err, "GetImage")
+		}
+
+		err = self.ValidateImage(ctx, image)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(image.ExternalId) > 0 && len(input.InstanceType) > 0 {
+			if cachedimage.UEFI.IsFalse() {
+				if strings.HasPrefix(input.InstanceType, "Standard_M") && strings.HasSuffix(input.InstanceType, "v2") {
+					return nil, httperrors.NewNotSupportedError("Azure Mv2-series instance sku only support UEFI image")
 				}
-			}
-			if !find {
-				return nil, httperrors.NewResourceNotFoundError("failed to find subformat vhd for image %s, please append 'vhd' for glance options(target_image_formats)", image.Name)
-			}
-		} else {
-			if len(input.InstanceType) > 0 {
-				if image.UEFI.IsFalse() {
-					if strings.HasPrefix(input.InstanceType, "Standard_M") && strings.HasSuffix(input.InstanceType, "v2") {
-						return nil, httperrors.NewNotSupportedError("Azure Mv2-series instance sku only support UEFI image")
-					}
-				} else {
-					// https://docs.microsoft.com/en-us/azure/virtual-machines/windows/generation-2
-					if !(strings.HasPrefix(input.InstanceType, "Standard_B") || // B-series
-						(strings.HasPrefix(input.InstanceType, "Standard_DC") && strings.HasSuffix(input.InstanceType, "s_v2") || input.InstanceType == "Standard_DC8_v2") || // DCsv2-series
-						(strings.HasPrefix(input.InstanceType, "Standard_DS") && strings.HasSuffix(input.InstanceType, "v2")) || // DSv2-series
-						(strings.HasPrefix(input.InstanceType, "Standard_DS") && strings.HasSuffix(input.InstanceType, "s_v3")) || // Dsv3-series
-						(strings.HasPrefix(input.InstanceType, "Standard_D") && strings.HasSuffix(input.InstanceType, "as_v4")) || // Dasv4-series
-						(strings.HasPrefix(input.InstanceType, "Standard_E") && strings.HasSuffix(input.InstanceType, "s_v3")) || // Esv3-series
-						(strings.HasPrefix(input.InstanceType, "Standard_E") && strings.HasSuffix(input.InstanceType, "as_v4")) || // Easv4-series
-						(strings.HasPrefix(input.InstanceType, "Standard_F") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Fsv2-series
-						(strings.HasPrefix(input.InstanceType, "Standard_GS")) || // GS-series
-						(strings.HasPrefix(input.InstanceType, "Standard_HB")) || // HB-series
-						(strings.HasPrefix(input.InstanceType, "Standard_HC")) || // HC-series
-						(strings.HasPrefix(input.InstanceType, "Standard_L") && strings.HasSuffix(input.InstanceType, "s")) || // Ls-series
-						(strings.HasPrefix(input.InstanceType, "Standard_L") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Ls-series
-						(strings.HasPrefix(input.InstanceType, "Standard_M")) || // M-series
-						(strings.HasPrefix(input.InstanceType, "Standard_M") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Mv2-series
-						(strings.HasPrefix(input.InstanceType, "Standard_NC") && strings.HasSuffix(input.InstanceType, "s_v2")) || // NCv2-series
-						(strings.HasPrefix(input.InstanceType, "Standard_NC") && strings.HasSuffix(input.InstanceType, "s_v3")) || // NCv3-series
-						(strings.HasPrefix(input.InstanceType, "Standard_ND")) || // ND-series
-						(strings.HasPrefix(input.InstanceType, "Standard_NV") && strings.HasSuffix(input.InstanceType, "s_v3"))) { // NVv3-series
-						return nil, httperrors.NewUnsupportOperationError("Azure UEFI image %s not support this instance sku", image.Name)
-					}
+			} else {
+				// https://docs.microsoft.com/en-us/azure/virtual-machines/windows/generation-2
+				if !(strings.HasPrefix(input.InstanceType, "Standard_B") || // B-series
+					(strings.HasPrefix(input.InstanceType, "Standard_DC") && strings.HasSuffix(input.InstanceType, "s_v2") || input.InstanceType == "Standard_DC8_v2") || // DCsv2-series
+					(strings.HasPrefix(input.InstanceType, "Standard_DS") && strings.HasSuffix(input.InstanceType, "v2")) || // DSv2-series
+					(strings.HasPrefix(input.InstanceType, "Standard_DS") && strings.HasSuffix(input.InstanceType, "s_v3")) || // Dsv3-series
+					(strings.HasPrefix(input.InstanceType, "Standard_D") && strings.HasSuffix(input.InstanceType, "as_v4")) || // Dasv4-series
+					(strings.HasPrefix(input.InstanceType, "Standard_E") && strings.HasSuffix(input.InstanceType, "s_v3")) || // Esv3-series
+					(strings.HasPrefix(input.InstanceType, "Standard_E") && strings.HasSuffix(input.InstanceType, "as_v4")) || // Easv4-series
+					(strings.HasPrefix(input.InstanceType, "Standard_F") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Fsv2-series
+					(strings.HasPrefix(input.InstanceType, "Standard_GS")) || // GS-series
+					(strings.HasPrefix(input.InstanceType, "Standard_HB")) || // HB-series
+					(strings.HasPrefix(input.InstanceType, "Standard_HC")) || // HC-series
+					(strings.HasPrefix(input.InstanceType, "Standard_L") && strings.HasSuffix(input.InstanceType, "s")) || // Ls-series
+					(strings.HasPrefix(input.InstanceType, "Standard_L") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Ls-series
+					(strings.HasPrefix(input.InstanceType, "Standard_M")) || // M-series
+					(strings.HasPrefix(input.InstanceType, "Standard_M") && strings.HasSuffix(input.InstanceType, "s_v2")) || // Mv2-series
+					(strings.HasPrefix(input.InstanceType, "Standard_NC") && strings.HasSuffix(input.InstanceType, "s_v2")) || // NCv2-series
+					(strings.HasPrefix(input.InstanceType, "Standard_NC") && strings.HasSuffix(input.InstanceType, "s_v3")) || // NCv3-series
+					(strings.HasPrefix(input.InstanceType, "Standard_ND")) || // ND-series
+					(strings.HasPrefix(input.InstanceType, "Standard_NV") && strings.HasSuffix(input.InstanceType, "s_v3"))) { // NVv3-series
+					return nil, httperrors.NewUnsupportOperationError("Azure UEFI image %s not support this instance sku", image.Name)
 				}
 			}
 		}

pkg/compute/guestdrivers/base.go

@@ -89,6 +89,10 @@ func (self *SBaseGuestDriver) StartDeleteGuestTask(ctx context.Context, userCred
 	return nil
 }
 
+func (self *SBaseGuestDriver) ValidateImage(ctx context.Context, image *cloudprovider.SImage) error {
+	return nil
+}
+
 func (self *SBaseGuestDriver) RequestDetachDisksFromGuestForDelete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
 	task.ScheduleRun(nil)
 	return nil

pkg/compute/guestdrivers/esxi.go

@@ -172,7 +172,7 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 	if img.ImageType != cloudprovider.CachedImageTypeSystem {
 		return desc
 	}
-	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).SubQuery()
+	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_READY).SubQuery()
 	scQ := models.StoragecacheManager.Query().In("id", sciSubQ)
 	storageCaches := make([]models.SStoragecache, 0, 1)
 	err = db.FetchModelObjects(models.StoragecacheManager, scQ, &storageCaches)
@@ -234,10 +234,32 @@ func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gues
 	}
 	config.Add(jsonutils.NewString(extId), "guest_ext_id")
 
-	accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId)
+	account := host.GetCloudaccount()
+	accessInfo, err := account.GetVCenterAccessInfo(storage.ExternalId)
 	if err != nil {
 		return err
 	}
+
+	action, _ := config.GetString("action")
+	if action == "create" {
+		project, err := db.TenantCacheManager.FetchTenantById(ctx, guest.ProjectId)
+		if err != nil {
+			return errors.Wrapf(err, "FetchTenantById(%s)", guest.ProjectId)
+		}
+
+		projects, err := account.GetExternalProjectsByProjectIdOrName(project.Id, project.Name)
+		if err != nil {
+			return errors.Wrapf(err, "GetExternalProjectsByProjectIdOrName(%s,%s)", project.Id, project.Name)
+		}
+
+		extProj := account.GetAvailableExternalProject(project, projects)
+		if extProj != nil {
+			config.Add(jsonutils.NewString(extProj.Name), "desc", "resource_pool")
+		} else {
+			config.Add(jsonutils.NewString(project.Name), "desc", "resource_pool")
+		}
+	}
+
 	config.Add(jsonutils.Marshal(accessInfo), "datastore")
 
 	url := "/disks/agent/deploy"

pkg/compute/guestdrivers/managedvirtual.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/osprofile"
 	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
 
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -320,7 +321,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context
 
 		secgroups := guest.GetSecgroups()
 		for i, secgroup := range secgroups {
-			externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup)
+			externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, desc.ProjectId)
 			if err != nil {
 				return errors.Wrap(err, "RequestSyncSecurityGroup")
 			}
@@ -449,7 +450,9 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte
 		}
 
 		if hostId := iVM.GetIHostId(); len(hostId) > 0 {
-			host, err := db.FetchByExternalId(models.HostManager, hostId)
+			host, err := db.FetchByExternalIdAndManagerId(models.HostManager, hostId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", host.ManagerId)
+			})
 			if err != nil {
 				log.Warningf("failed to found new hostId(%s) for ivm %s(%s) error: %v", hostId, guest.Name, guest.Id, err)
 			} else if host.GetId() != guest.HostId {
@@ -886,7 +889,13 @@ func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx co
 				}
 
 				if len(diskInfo[i].StorageExternalId) > 0 {
-					storage, err := db.FetchByExternalId(models.StorageManager, diskInfo[i].StorageExternalId)
+					storage, err := db.FetchByExternalIdAndManagerId(models.StorageManager, diskInfo[i].StorageExternalId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+						host := guest.GetHost()
+						if host != nil {
+							return q.Equals("manager_id", host.ManagerId)
+						}
+						return q
+					})
 					if err != nil {
 						log.Warningf("failed to found storage by externalId %s error: %v", diskInfo[i].StorageExternalId, err)
 					} else if disk.StorageId != storage.GetId() {
@@ -967,10 +976,19 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncSecgroupsOnHost(ctx conte
 		return errors.Wrap(err, "GetSecurityGroupVpcId")
 	}
 
+	remoteProjectId := ""
+	provider := host.GetCloudprovider()
+	if provider != nil {
+		remoteProjectId, err = provider.SyncProject(ctx, task.GetUserCred(), guest.ProjectId)
+		if err != nil {
+			log.Errorf("failed to sync project %s for guest %s error: %v", guest.ProjectId, guest.Name, err)
+		}
+	}
+
 	secgroups := guest.GetSecgroups()
 	externalIds := []string{}
 	for _, secgroup := range secgroups {
-		externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup)
+		externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, remoteProjectId)
 		if err != nil {
 			return errors.Wrap(err, "RequestSyncSecurityGroup")
 		}

pkg/compute/guestdrivers/openstack.go

@@ -114,6 +114,10 @@ func (self *SOpenStackGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudpr
 	return true
 }
 
+func (self *SOpenStackGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
 func (self *SOpenStackGuestDriver) IsNeedRestartForResetLoginInfo() bool {
 	return false
 }

pkg/compute/hostdrivers/managedvirtual.go

@@ -275,7 +275,7 @@ func (self *SManagedVirtualizationHostDriver) RequestAllocateDiskOnStorage(ctx c
 
 		cloudprovider.WaitStatus(iDisk, api.DISK_READY, time.Second*5, time.Minute*5)
 
-		models.SyncMetadata(ctx, task.GetUserCred(), disk, iDisk)
+		models.SyncVirtualResourceMetadata(ctx, task.GetUserCred(), disk, iDisk)
 
 		data := jsonutils.NewDict()
 		data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size")

pkg/compute/models/buckets.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -420,6 +421,16 @@ func (manager *SBucketManager) ValidateCreateData(
 		return input, httperrors.NewInputParameterError("invalid bucket name %s: %s", input.Name, err)
 	}
 
+	if len(input.StorageClass) > 0 {
+		driver, err := managerV.GetProvider()
+		if err != nil {
+			return input, errors.Wrap(err, "GetProvider")
+		}
+		if !utils.IsInStringArray(input.StorageClass, driver.GetStorageClasses(cloudRegionV.Id)) {
+			return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", input.StorageClass)
+		}
+	}
+
 	quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, cloudRegionV, managerV)
 	pendingUsage := SRegionQuota{Bucket: 1}
 	pendingUsage.SetKeys(quotaKeys)
@@ -956,14 +967,17 @@ func (bucket *SBucket) PerformUpload(
 		return nil, httperrors.NewInputParameterError("Content-Length negative %d", sizeBytes)
 	}
 	storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
+	driver, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
+	}
+	if len(storageClass) > 0 && !utils.IsInStringArray(storageClass, driver.GetStorageClasses(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", storageClass)
+	}
+
 	aclStr := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)
-	if len(aclStr) > 0 {
-		switch cloudprovider.TBucketACLType(aclStr) {
-		case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
-			// do nothing
-		default:
-			return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
-		}
+	if len(aclStr) > 0 && !utils.IsInStringArray(aclStr, driver.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid acl %s", aclStr)
 	}
 
 	inc := cloudprovider.SBucketStats{}
@@ -1038,15 +1052,23 @@ func (bucket *SBucket) PerformAcl(
 ) (jsonutils.JSONObject, error) {
 	err := input.Validate()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "ValidateInput")
+	}
+
+	provider, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
 	}
 
 	iBucket, objects, err := bucket.processObjectsActionInput(input.BucketObjectsActionInput)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "processObjectsActionInput")
 	}
 
 	if len(objects) == 0 {
+		if !utils.IsInStringArray(string(input.Acl), provider.GetBucketCannedAcls(bucket.CloudregionId)) {
+			return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported bucket canned acl %s", input.Acl)
+		}
 		err = iBucket.SetAcl(input.Acl)
 		if err != nil {
 			return nil, httperrors.NewInternalServerError("setAcl error %s", err)
@@ -1059,6 +1081,10 @@ func (bucket *SBucket) PerformAcl(
 		return nil, nil
 	}
 
+	if !utils.IsInStringArray(string(input.Acl), provider.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported object canned acl %s", input.Acl)
+	}
+
 	errs := make([]error, 0)
 	for _, object := range objects {
 		err := object.SetAcl(input.Acl)

pkg/compute/models/cachedimages.go

@@ -175,7 +175,9 @@ func (self *SCachedimage) getStoragecacheCount() (int, error) {
 }
 
 func (self *SCachedimage) GetImage() (*cloudprovider.SImage, error) {
-	image := cloudprovider.SImage{}
+	image := cloudprovider.SImage{
+		ExternalId: self.ExternalId,
+	}
 
 	err := self.Info.Unmarshal(&image)
 	if err != nil {

pkg/compute/models/capabilities.go

@@ -315,10 +315,10 @@ type StorageInfo struct {
 	Name            string
 	VirtualCapacity int64
 	Capacity        int64
-	Reserved        int64
+	Reserved        sql.NullInt64
 	StorageType     string
 	MediumType      string
-	Cmtbound        float32
+	Cmtbound        sql.NullFloat64
 	UsedCapacity    sql.NullInt64
 	WasteCapacity   sql.NullInt64
 	FreeCapacity    int64
@@ -440,7 +440,7 @@ func getStorageTypes(
 		addStorageInfo = func(storage *StorageInfo, simpleStorage *SimpleStorageInfo) {
 			simpleStorage.VirtualCapacity += storage.VirtualCapacity
 			simpleStorage.FreeCapacity += storage.FreeCapacity
-			simpleStorage.Reserved += storage.Reserved
+			simpleStorage.Reserved += storage.Reserved.Int64
 			simpleStorage.Capacity += storage.Capacity
 			simpleStorage.WasteCapacity += storage.WasteCapacity.Int64
 			simpleStorage.UsedCapacity += storage.UsedCapacity.Int64
@@ -493,10 +493,10 @@ func getStorageTypes(
 				}
 				allStorageTypes = append(allStorageTypes, storageType)
 			}
-			if storage.Cmtbound == 0 {
-				storage.Cmtbound = options.Options.DefaultStorageOvercommitBound
+			if storage.Cmtbound.Float64 == 0 {
+				storage.Cmtbound.Float64 = float64(options.Options.DefaultStorageOvercommitBound)
 			}
-			storage.VirtualCapacity = int64(float32((storage.Capacity - storage.Reserved)) * storage.Cmtbound)
+			storage.VirtualCapacity = int64(float64(storage.Capacity-storage.Reserved.Int64) * storage.Cmtbound.Float64)
 			storage.FreeCapacity = storage.VirtualCapacity - storage.UsedCapacity.Int64 - storage.WasteCapacity.Int64
 			addStorageInfo(&storage, simpleStorage)
 			storageInfos[storageType] = simpleStorage

pkg/compute/models/cloudaccounts.go

@@ -21,6 +21,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -34,7 +35,6 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
@@ -46,7 +46,6 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -341,17 +340,37 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		return input, err
 	}
 
-	if len(input.Project) > 0 {
-		_, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
-		if err != nil {
-			return input, errors.Wrap(err, "db.ValidateProjectizedResourceInput")
-		}
-	}
-
 	if !cloudprovider.IsSupported(input.Provider) {
 		return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider)
 	}
 	providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider)
+
+	forceAutoCreateProject := providerDriver.IsNeedForceAutoCreateProject()
+	if forceAutoCreateProject {
+		input.AutoCreateProject = &forceAutoCreateProject
+	}
+
+	if len(input.Project) > 0 {
+		var proj *db.STenant
+		proj, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
+		if err != nil {
+			return input, errors.Wrap(err, "db.ValidateProjectizedResourceInput")
+		}
+		if proj.DomainId != ownerId.GetProjectDomainId() {
+			return input, httperrors.NewInputParameterError("Project %s(%s) not belong to domain %s(%s)", proj.Name, proj.Id, ownerId.GetProjectDomain(), ownerId.GetProjectDomainId())
+		}
+		if input.AutoCreateProject != nil && *input.AutoCreateProject {
+			log.Warningf("project_id and auto_create_project should not be turned on at the same time")
+		}
+		if !forceAutoCreateProject {
+			input.AutoCreateProject = nil
+		}
+	} else if input.AutoCreateProject == nil || !*input.AutoCreateProject {
+		log.Warningf("auto_create_project is off while no project_id specified")
+		createProject := true
+		input.AutoCreateProject = &createProject
+	}
+
 	input.SCloudaccount, err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, input.SCloudaccountCredential)
 	if err != nil {
 		return input, err
@@ -429,20 +448,6 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		input.SyncIntervalSeconds = options.Options.MinimalSyncIntervalSeconds
 	}
 
-	if !input.AutoCreateProject {
-		if userCred.GetProjectDomainId() != ownerId.GetProjectDomainId() {
-			s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
-			params := jsonutils.Marshal(map[string]string{"domain_id": ownerId.GetProjectDomainId()})
-			tenants, err := modules.Projects.List(s, params)
-			if err != nil {
-				return input, err
-			}
-			if tenants.Total == 0 {
-				return input, httperrors.NewInputParameterError("There is no projects under the domain %s", ownerId.GetProjectDomainId())
-			}
-		}
-	}
-
 	input.EnabledStatusInfrasResourceBaseCreateInput, err = manager.SEnabledStatusInfrasResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusInfrasResourceBaseCreateInput)
 	if err != nil {
 		return input, err
@@ -728,9 +733,9 @@ func (self *SCloudaccount) MarkEndSyncWithLock(ctx context.Context, userCred mcc
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
-	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
-		return nil
-	}
+	// if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
+	// 	return nil
+	// }
 
 	providers := self.GetCloudproviders()
 	for i := range providers {
@@ -1742,6 +1747,18 @@ func (manager *SCloudaccountManager) initAllRecords() {
 	}
 }
 
+func (self *SCloudaccount) CanSync() bool {
+	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING || self.getSyncStatus2() == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING {
+		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second {
+			return true
+		} else {
+			return false
+		}
+	} else {
+		return true
+	}
+}
+
 func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 	if isStart && !options.Options.IsSlaveNode {
 		// mark all the records to be idle
@@ -1777,7 +1794,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	manager, err := account.getProviderInternal()
 	if err != nil {
 		log.Errorf("account.GetProvider failed: %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "account.getProviderInternal")
 	}
 	balance, status, err := manager.GetBalance()
 	if err != nil {
@@ -1795,7 +1812,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	sysInfo, err := manager.GetSysInfo()
 	if err != nil {
 		log.Errorf("manager.GetSysInfo fail %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "manager.GetSysInfo")
 	}
 	factory := manager.GetFactory()
 	diff, err := db.Update(account, func() error {
@@ -1848,7 +1865,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 	if err != nil {
 		account.markAllProvidersDicconnected(ctx, userCred)
 		account.markAccountDiscconected(ctx, userCred)
-		return err
+		return errors.Wrap(err, "account.probeAccountStatus")
 	}
 	account.markAccountConnected(ctx, userCred)
 	providers := account.importAllSubaccounts(ctx, userCred, subaccounts)
@@ -1857,7 +1874,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 			_, err := providers[i].prepareCloudproviderRegions(ctx, userCred)
 			if err != nil {
 				log.Errorf("syncCloudproviderRegion fail %s", err)
-				return err
+				return errors.Wrap(err, "providers[i].prepareCloudproviderRegions")
 			}
 		}
 	}
@@ -1876,13 +1893,39 @@ func (account *SCloudaccount) markAutoSync(userCred mcclient.TokenCredential) er
 	return nil
 }
 
+var (
+	cloudaccountPendingSyncs      = map[string]struct{}{}
+	cloudaccountPendingSyncsMutex = &sync.Mutex{}
+)
+
 func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCred mcclient.TokenCredential, waitChan chan error, autoSync bool) {
+	cloudaccountPendingSyncsMutex.Lock()
+	defer cloudaccountPendingSyncsMutex.Unlock()
+	if _, ok := cloudaccountPendingSyncs[account.Id]; ok {
+		if waitChan != nil {
+			go func() {
+				// an active cloudaccount sync task is running, return with conflict error
+				log.Errorf("an active cloudaccount sync task is running, early return with conflict error")
+				waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs")
+			}()
+		}
+		return
+	}
+	cloudaccountPendingSyncs[account.Id] = struct{}{}
+
 	RunSyncCloudAccountTask(func() {
+		func() {
+			cloudaccountPendingSyncsMutex.Lock()
+			defer cloudaccountPendingSyncsMutex.Unlock()
+			delete(cloudaccountPendingSyncs, account.Id)
+		}()
+
 		log.Debugf("syncAccountStatus %s %s", account.Id, account.Name)
 		err := account.syncAccountStatus(ctx, userCred)
 		if waitChan != nil {
 			if err != nil {
 				account.markEndSync(userCred)
+				err = errors.Wrap(err, "account.syncAccountStatus")
 			}
 			waitChan <- err
 		} else {
@@ -1892,7 +1935,8 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 				account.markAutoSync(userCred)
 				providers := account.GetEnabledCloudproviders()
 				for i := range providers {
-					providers[i].syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
+					provider := &providers[i]
+					provider.syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
 					syncCnt += 1
 				}
 			}
@@ -1918,6 +1962,16 @@ func (self *SCloudaccount) Delete(ctx context.Context, userCred mcclient.TokenCr
 
 func (self *SCloudaccount) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	self.SetStatus(userCred, api.CLOUD_PROVIDER_DELETED, "real delete")
+	projects, err := self.GetExternalProjects()
+	if err != nil {
+		return errors.Wrap(err, "GetExternalProjects")
+	}
+	for i := range projects {
+		err = projects[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "project %s Delete", projects[i].Id)
+		}
+	}
 	return self.SEnabledStatusInfrasResourceBase.Delete(ctx, userCred)
 }
 
@@ -2248,6 +2302,32 @@ func (account *SCloudaccount) PerformSyncSkus(ctx context.Context, userCred mccl
 	return nil, nil
 }
 
+func (self *SCloudaccount) GetExternalProjects() ([]SExternalProject, error) {
+	projects := []SExternalProject{}
+	q := ExternalProjectManager.Query().Equals("cloudaccount_id", self.Id)
+	err := db.FetchModelObjects(ExternalProjectManager, q, &projects)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return projects, nil
+}
+
+func (self *SCloudaccount) GetExternalProjectsByProjectIdOrName(projectId, name string) ([]SExternalProject, error) {
+	projects := []SExternalProject{}
+	q := ExternalProjectManager.Query().Equals("cloudaccount_id", self.Id)
+	q = q.Filter(
+		sqlchemy.OR(
+			sqlchemy.Equals(q.Field("name"), name),
+			sqlchemy.Equals(q.Field("tenant_id"), projectId),
+		),
+	)
+	err := db.FetchModelObjects(ExternalProjectManager, q, &projects)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return projects, nil
+}
+
 func (manager *SCloudaccountManager) queryCloudAccountByCapability(region *SCloudregion, zone *SZone, domainId string, enabled tristate.TriState, capability string) *sqlchemy.SQuery {
 	providers := CloudproviderManager.Query().SubQuery()
 	q := manager.Query()
@@ -2339,3 +2419,78 @@ func (account *SCloudaccount) GetUsages() []db.IUsage {
 		&usage,
 	}
 }
+
+func (self *SCloudaccount) GetAvailableExternalProject(local *db.STenant, projects []SExternalProject) *SExternalProject {
+	var ret *SExternalProject = nil
+	for i := 0; i < len(projects); i++ {
+		if projects[i].Status == api.EXTERNAL_PROJECT_STATUS_AVAILABLE {
+			if projects[i].ProjectId == local.Id {
+				return &projects[i]
+			}
+			if projects[i].Name == local.Name {
+				ret = &projects[i]
+			}
+		}
+	}
+	return ret
+}
+
+// 若本地项目映射了多个云上项目，则在云上随机找一个项目
+// 若本地项目没有映射云上任何项目，则在云上新建一个同名项目
+// 若本地项目a映射云上项目b，但b项目不可用,则看云上是否有a项目，有则直接使用,若没有则在云上创建a-1, a-2类似项目
+func (self *SCloudaccount) SyncProject(ctx context.Context, userCred mcclient.TokenCredential, projectId string) (string, error) {
+	lockman.LockRawObject(ctx, self.Id, projectId)
+	defer lockman.ReleaseRawObject(ctx, self.Id, projectId)
+
+	provider, err := self.GetProvider()
+	if err != nil {
+		return "", errors.Wrap(err, "GetProvider")
+	}
+
+	if !cloudprovider.IsSupportProject(provider) {
+		return "", nil
+	}
+
+	project, err := db.TenantCacheManager.FetchTenantById(ctx, projectId)
+	if err != nil {
+		return "", errors.Wrapf(err, "FetchTenantById(%s)", projectId)
+	}
+
+	projects, err := self.GetExternalProjectsByProjectIdOrName(projectId, project.Name)
+	if err != nil {
+		return "", errors.Wrapf(err, "GetExternalProjectsByProjectIdOrName(%s,%s)", projectId, project.Name)
+	}
+
+	extProj := self.GetAvailableExternalProject(project, projects)
+	if extProj != nil {
+		return extProj.ExternalId, nil
+	}
+
+	retry := 1
+	if len(projects) > 0 {
+		retry = 10
+	}
+
+	var iProject cloudprovider.ICloudProject = nil
+	projectName := project.Name
+	for i := 0; i < retry; i++ {
+		iProject, err = provider.CreateIProject(projectName)
+		if err == nil {
+			break
+		}
+		projectName = fmt.Sprintf("%s-%d", project.Name, i)
+	}
+	if err != nil {
+		if errors.Cause(err) != cloudprovider.ErrNotImplemented && errors.Cause(err) != cloudprovider.ErrNotSupported {
+			logclient.AddSimpleActionLog(self, logclient.ACT_CREATE, err, userCred, false)
+		}
+		return "", errors.Wrapf(err, "CreateIProject(%s)", projectName)
+	}
+
+	extProj, err = ExternalProjectManager.newFromCloudProject(ctx, userCred, self, project, iProject)
+	if err != nil {
+		return "", errors.Wrap(err, "newFromCloudProject")
+	}
+
+	return extProj.ExternalId, nil
+}

pkg/compute/models/cloudproviderregions.go

@@ -122,7 +122,7 @@ func (self *SCloudproviderregion) GetAccount() *SCloudaccount {
 func (self *SCloudproviderregion) GetRegion() *SCloudregion {
 	regionObj, err := CloudregionManager.FetchById(self.CloudregionId)
 	if err != nil {
-		log.Errorf("CloudproviderManager.FetchById fail %s", err)
+		log.Errorf("CloudregionManager.FetchById(%s) fail %s", self.CloudregionId, err)
 		return nil
 	}
 	return regionObj.(*SCloudregion)

pkg/compute/models/cloudproviders.go

@@ -1128,7 +1128,8 @@ func (manager *SCloudproviderManager) ListItemFilter(
 			sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")),
 		))
 		if zone != nil {
-			sq = sq.Filter(sqlchemy.Equals(wires.Field("zone_id"), zone.GetId()))
+			zoneFilter := sqlchemy.OR(sqlchemy.Equals(wires.Field("zone_id"), zone.GetId()), sqlchemy.IsNullOrEmpty(wires.Field("zone_id")))
+			sq = sq.Filter(zoneFilter)
 		} else if region != nil {
 			sq = sq.Filter(sqlchemy.Equals(vpcs.Field("cloudregion_id"), region.GetId()))
 		}
@@ -1372,7 +1373,6 @@ func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.To
 		ElasticipManager,
 		NetworkInterfaceManager,
 		CloudproviderRegionManager,
-		ExternalProjectManager,
 		CloudregionManager,
 		CloudproviderQuotaManager,
 	} {
@@ -1405,6 +1405,14 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	return nil
 }
 
+func (self *SCloudprovider) GetRegionDriver() (IRegionDriver, error) {
+	driver := GetRegionDriver(self.Provider)
+	if driver == nil {
+		return nil, fmt.Errorf("failed to found region driver for %s", self.Provider)
+	}
+	return driver, nil
+}
+
 func (self *SCloudprovider) ClearSchedDescCache() error {
 	hosts := make([]SHost, 0)
 	q := HostManager.Query().Equals("manager_id", self.Id)
@@ -1555,33 +1563,56 @@ func (provider *SCloudprovider) AllowGetDetailsStorageClasses(
 func (provider *SCloudprovider) GetDetailsStorageClasses(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
-	query jsonutils.JSONObject,
-) (jsonutils.JSONObject, error) {
+	input api.CloudproviderGetStorageClassInput,
+) (api.CloudproviderGetStorageClassOutput, error) {
+	output := api.CloudproviderGetStorageClassOutput{}
 	driver, err := provider.GetProvider()
 	if err != nil {
-		return nil, httperrors.NewInternalServerError("fail to get provider driver %s", err)
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
 	}
-	extId := ""
-	regionStr := jsonutils.GetAnyString(query, []string{"cloudregion", "cloudregion_id"})
-	if len(regionStr) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
 		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
 		}
-		extId = regionObj.(*SCloudregion).GetExternalId()
 	}
 
-	sc := driver.GetStorageClasses(extId)
+	sc := driver.GetStorageClasses(input.Cloudregion)
 	if sc == nil {
-		return nil, httperrors.NewInternalServerError("storage classes not supported")
+		return output, httperrors.NewInternalServerError("storage classes not supported")
 	}
-	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.NewStringArray(sc), "storage_classes")
-	return ret, nil
+	output.StorageClasses = sc
+	return output, nil
+}
+
+func (provider *SCloudprovider) AllowGetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) bool {
+	return db.IsAdminAllowGetSpec(userCred, provider, "canned-acls")
+}
+
+func (provider *SCloudprovider) GetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	input api.CloudproviderGetCannedAclInput,
+) (api.CloudproviderGetCannedAclOutput, error) {
+	output := api.CloudproviderGetCannedAclOutput{}
+	driver, err := provider.GetProvider()
+	if err != nil {
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
+	}
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
+		if err != nil {
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
+		}
+	}
+
+	output.BucketCannedAcls = driver.GetBucketCannedAcls(input.Cloudregion)
+	output.ObjectCannedAcls = driver.GetObjectCannedAcls(input.Cloudregion)
+	return output, nil
 }
 
 func (provider *SCloudprovider) getAccountShareInfo() apis.SAccountShareInfo {
@@ -1620,51 +1651,10 @@ func (provider *SCloudprovider) GetChangeOwnerCandidateDomainIds() []string {
 	return []string{}
 }
 
-func (self *SCloudprovider) GetExternalProjects() ([]SExternalProject, error) {
-	q := ExternalProjectManager.Query().Equals("manager_id", self.Id)
-	projects := []SExternalProject{}
-	err := db.FetchModelObjects(ExternalProjectManager, q, &projects)
-	if err != nil {
-		return nil, errors.Wrap(err, "FetchModelObjects")
-	}
-	return projects, nil
-}
-
 func (self *SCloudprovider) SyncProject(ctx context.Context, userCred mcclient.TokenCredential, id string) (string, error) {
-	lockman.LockRawObject(ctx, self.Id, id)
-	defer lockman.ReleaseRawObject(ctx, self.Id, id)
-
-	projects, err := self.GetExternalProjects()
-	if err != nil {
-		return "", errors.Wrap(err, "GetExternalProjects")
+	account := self.GetCloudaccount()
+	if account == nil {
+		return "", fmt.Errorf("failed to get cloudprovider %s account", self.Name)
 	}
-	for _, project := range projects {
-		if project.ProjectId == id {
-			return project.ExternalId, nil
-		}
-	}
-
-	project, err := db.TenantCacheManager.FetchById(id)
-	if err != nil {
-		return "", errors.Wrap(err, "TenantCacheManager.FetchById")
-	}
-
-	for _, extProj := range projects {
-		if extProj.Name == project.GetName() {
-			return extProj.ExternalId, nil
-		}
-	}
-	provider, err := self.GetProvider()
-	if err != nil {
-		return "", errors.Wrap(err, "GetProvider")
-	}
-	iProject, err := provider.CreateIProject(project.GetName())
-	if err != nil {
-		return "", errors.Wrap(err, "CreateIProject")
-	}
-	extProj, err := ExternalProjectManager.newFromCloudProject(ctx, userCred, self, iProject)
-	if err != nil {
-		return "", errors.Wrap(err, "newFromCloudProject")
-	}
-	return extProj.ExternalId, nil
+	return account.SyncProject(ctx, userCred, id)
 }

pkg/compute/models/cloudregions.go

@@ -226,15 +226,38 @@ func (self *SCloudregion) getGuestCountInternal(increment bool) (int, error) {
 	return query.CountWithError()
 }
 
-func (self *SCloudregion) GetVpcCount() (int, error) {
+func (self *SCloudregion) GetVpcQuery() *sqlchemy.SQuery {
 	vpcs := VpcManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {
 		return vpcs.Filter(sqlchemy.OR(sqlchemy.IsNull(vpcs.Field("cloudregion_id")),
 			sqlchemy.IsEmpty(vpcs.Field("cloudregion_id")),
-			sqlchemy.Equals(vpcs.Field("cloudregion_id"), self.Id))).CountWithError()
-	} else {
-		return vpcs.Equals("cloudregion_id", self.Id).CountWithError()
+			sqlchemy.Equals(vpcs.Field("cloudregion_id"), self.Id)))
 	}
+	return vpcs.Equals("cloudregion_id", self.Id)
+}
+
+func (self *SCloudregion) GetVpcCount() (int, error) {
+	return self.GetVpcQuery().CountWithError()
+}
+
+func (self *SCloudregion) GetVpcs() ([]SVpc, error) {
+	vpcs := []SVpc{}
+	q := self.GetVpcQuery()
+	err := db.FetchModelObjects(VpcManager, q, &vpcs)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return vpcs, nil
+}
+
+func (self *SCloudregion) GetCloudproviderVpcs(managerId string) ([]SVpc, error) {
+	vpcs := []SVpc{}
+	q := self.GetVpcQuery().Equals("manager_id", managerId)
+	err := db.FetchModelObjects(VpcManager, q, &vpcs)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return vpcs, nil
 }
 
 func (self *SCloudregion) GetDriver() IRegionDriver {
@@ -497,7 +520,7 @@ func (self *SCloudregion) AllowPerformDefaultVpc(ctx context.Context, userCred m
 }
 
 func (self *SCloudregion) PerformDefaultVpc(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	vpcs, err := VpcManager.getVpcsByRegion(self, nil)
+	vpcs, err := self.GetVpcs()
 	if err != nil {
 		return nil, err
 	}

pkg/compute/models/cloudsync.go

@@ -172,28 +172,6 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca
 	}
 }
 
-func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, driver cloudprovider.ICloudProvider, provider *SCloudprovider) {
-	projects, err := driver.GetIProjects()
-	if err != nil {
-		msg := fmt.Sprintf("GetIProjects for provider %s failed %s", provider.GetName(), err)
-		log.Errorf(msg)
-		// logSyncFailed(provider, task, msg)
-		return
-	}
-
-	result := ExternalProjectManager.SyncProjects(ctx, userCred, provider, projects)
-
-	syncResults.Add(ExternalProjectManager, result)
-
-	msg := result.Result()
-	log.Infof("SyncProjects for provider %s result: %s", provider.Name, msg)
-	if result.IsError() {
-		// logSyncFailed(provider, task, msg)
-		return
-	}
-	// db.OpsLog.LogEvent(provider, db.ACT_SYNC_PROJECT_COMPLETE, msg, task.UserCred)
-}
-
 func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) {
 	eips, err := remoteRegion.GetIEips()
 	if err != nil {
@@ -262,6 +240,10 @@ func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, sync
 			lockman.LockObject(ctx, &localVpcs[j])
 			defer lockman.ReleaseObject(ctx, &localVpcs[j])
 
+			if localVpcs[j].Deleted {
+				return
+			}
+
 			syncVpcWires(ctx, userCred, syncResults, provider, &localVpcs[j], remoteVpcs[j], syncRange)
 			if localRegion.GetDriver().IsSecurityGroupBelongVpc() || localRegion.GetDriver().IsSupportClassicSecurityGroup() || j == 0 { //有vpc属性的每次都同步,支持classic的vpc也同步，否则仅同步一次
 				syncVpcSecGroup(ctx, userCred, syncResults, provider, &localVpcs[j], remoteVpcs[j], syncRange)
@@ -333,6 +315,10 @@ func syncVpcNatgateways(ctx context.Context, userCred mcclient.TokenCredential,
 			lockman.LockObject(ctx, &localNatGateways[i])
 			defer lockman.ReleaseObject(ctx, &localNatGateways[i])
 
+			if localNatGateways[i].Deleted {
+				return
+			}
+
 			syncNatGatewayEips(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
 			syncNatDTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
 			syncNatSTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
@@ -414,6 +400,9 @@ func syncVpcWires(ctx context.Context, userCred mcclient.TokenCredential, syncRe
 			lockman.LockObject(ctx, &localWires[i])
 			defer lockman.ReleaseObject(ctx, &localWires[i])
 
+			if localWires[i].Deleted {
+				return
+			}
 			syncWireNetworks(ctx, userCred, syncResults, provider, &localWires[i], remoteWires[i], syncRange)
 		}()
 	}
@@ -468,6 +457,10 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy
 			lockman.LockObject(ctx, &localStorages[i])
 			defer lockman.ReleaseObject(ctx, &localStorages[i])
 
+			if localStorages[i].Deleted {
+				return
+			}
+
 			if !isInCache(storageCachePairs, localStorages[i].StoragecacheId) && !isInCache(newCacheIds, localStorages[i].StoragecacheId) {
 				cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i])
 				if cachePair.remote != nil && cachePair.local != nil {
@@ -553,6 +546,10 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR
 			lockman.LockObject(ctx, &localHosts[i])
 			defer lockman.ReleaseObject(ctx, &localHosts[i])
 
+			if localHosts[i].Deleted {
+				return
+			}
+
 			syncMetadata(ctx, userCred, &localHosts[i], remoteHosts[i])
 			newCachePairs = syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs)
 			syncHostWires(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i])
@@ -646,6 +643,10 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes
 			lockman.LockObject(ctx, syncVMPairs[i].Local)
 			defer lockman.ReleaseObject(ctx, syncVMPairs[i].Local)
 
+			if syncVMPairs[i].Local.Deleted || syncVMPairs[i].Local.PendingDeleted {
+				return
+			}
+
 			syncVMPeripherals(ctx, userCred, syncVMPairs[i].Local, syncVMPairs[i].Remote, localHost, provider, driver)
 			// syncMetadata(ctx, userCred, syncVMPairs[i].Local, syncVMPairs[i].Remote)
 			// syncVMNics(ctx, userCred, provider, localHost, syncVMPairs[i].Local, syncVMPairs[i].Remote)
@@ -658,7 +659,7 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes
 }
 
 func syncVMPeripherals(ctx context.Context, userCred mcclient.TokenCredential, local *SGuest, remote cloudprovider.ICloudVM, host *SHost, provider *SCloudprovider, driver cloudprovider.ICloudProvider) {
-	syncMetadata(ctx, userCred, local, remote)
+	syncVirtualResourceMetadata(ctx, userCred, local, remote)
 	err := syncVMNics(ctx, userCred, provider, host, local, remote)
 	if err != nil {
 		log.Errorf("syncVMNics error %s", err)
@@ -795,6 +796,10 @@ func syncRegionDBInstances(ctx context.Context, userCred mcclient.TokenCredentia
 			lockman.LockObject(ctx, &localInstances[i])
 			defer lockman.ReleaseObject(ctx, &localInstances[i])
 
+			if localInstances[i].Deleted || localInstances[i].PendingDeleted {
+				return
+			}
+
 			syncDBInstanceNetwork(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
 			syncDBInstanceParameters(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
 			syncDBInstanceDatabases(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
@@ -902,6 +907,10 @@ func syncDBInstanceAccounts(ctx context.Context, userCred mcclient.TokenCredenti
 			lockman.LockObject(ctx, &localAccounts[i])
 			defer lockman.ReleaseObject(ctx, &localAccounts[i])
 
+			if localAccounts[i].Deleted {
+				return
+			}
+
 			syncDBInstanceAccountPrivileges(ctx, userCred, syncResults, &localAccounts[i], remoteAccounts[i])
 
 		}()
@@ -985,6 +994,10 @@ func syncRegionNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCre
 			lockman.LockObject(ctx, &localInterfaces[i])
 			defer lockman.ReleaseObject(ctx, &localInterfaces[i])
 
+			if localInterfaces[i].Deleted {
+				return
+			}
+
 			syncInterfaceAddresses(ctx, userCred, &localInterfaces[i], remoteInterfaces[i])
 		}()
 	}
@@ -1027,10 +1040,6 @@ func syncPublicCloudProviderInfo(
 
 	storageCachePairs := make([]sStoragecacheSyncPair, 0)
 
-	if cloudprovider.IsSupportProject(driver) {
-		syncProjects(ctx, userCred, syncResults, driver, provider)
-	}
-
 	syncRegionQuotas(ctx, userCred, syncResults, driver, provider, localRegion, remoteRegion)
 
 	localZones, remoteZones, _ := syncRegionZones(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
@@ -1124,10 +1133,6 @@ func syncOnPremiseCloudProviderInfo(
 ) error {
 	log.Debugf("Start sync on-premise provider %s(%s)", provider.Name, provider.Provider)
 
-	if cloudprovider.IsSupportProject(driver) {
-		syncProjects(ctx, userCred, syncResults, driver, provider)
-	}
-
 	iregion, err := driver.GetOnPremiseIRegion()
 	if err != nil {
 		msg := fmt.Sprintf("GetOnPremiseIRegion for provider %s failed %s", provider.GetName(), err)
@@ -1171,15 +1176,17 @@ func syncOnPremiseCloudProviderInfo(
 		}
 	}
 
-	log.Debugf("storageCachePairs count %d", len(storageCachePairs))
-	for i := range storageCachePairs {
-		// alway sync on-premise cached images
-		// if storageCachePairs[i].isNew || syncRange.DeepSync {
-		result := storageCachePairs[i].syncCloudImages(ctx, userCred)
-		syncResults.Add(StoragecachedimageManager, result)
-		msg := result.Result()
-		log.Infof("syncCloudImages result: %s", msg)
-		// }
+	if cloudprovider.IsSupportCompute(driver) {
+		log.Debugf("storageCachePairs count %d", len(storageCachePairs))
+		for i := range storageCachePairs {
+			// alway sync on-premise cached images
+			// if storageCachePairs[i].isNew || syncRange.DeepSync {
+			result := storageCachePairs[i].syncCloudImages(ctx, userCred)
+			syncResults.Add(StoragecachedimageManager, result)
+			msg := result.Result()
+			log.Infof("syncCloudImages result: %s", msg)
+			// }
+		}
 	}
 
 	return nil
@@ -1221,7 +1228,7 @@ func SyncCloudProject(userCred mcclient.TokenCredential, model db.IVirtualModel,
 	if extProjectId := extModel.GetProjectId(); len(extProjectId) > 0 {
 		extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId)
 		if err != nil {
-			log.Errorln(err)
+			log.Errorf("sync project for %s %s error: %v", model.Keyword(), model.GetName(), err)
 		} else {
 			newOwnerId = extProject.GetOwnerId()
 		}

pkg/compute/models/dbinstance_backups.go

@@ -259,11 +259,16 @@ func (self *SDBInstanceBackup) PostCreate(ctx context.Context, userCred mcclient
 }
 
 func (self *SDBInstanceBackup) StartDBInstanceBackupCreateTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error {
-	self.SetStatus(userCred, api.DBINSTANCE_BACKUP_CREATING, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "DBInstanceBackupCreateTask", self, userCred, data, parentTaskId, "", nil)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "NewTask")
 	}
+	instance, err := self.GetDBInstance()
+	if err != nil {
+		return errors.Wrap(err, "GetDBInstance")
+	}
+	instance.SetStatus(userCred, api.DBINSTANCE_BACKING_UP, "")
+	self.SetStatus(userCred, api.DBINSTANCE_BACKUP_CREATING, "")
 	task.ScheduleRun(nil)
 	return nil
 }
@@ -437,7 +442,9 @@ func (self *SDBInstanceBackup) SyncWithCloudDBInstanceBackup(
 
 		if dbinstanceId := extBackup.GetDBInstanceId(); len(dbinstanceId) > 0 {
 			//有可能云上删除了实例，未删除备份
-			_instance, err := db.FetchByExternalId(DBInstanceManager, dbinstanceId)
+			_instance, err := db.FetchByExternalIdAndManagerId(DBInstanceManager, dbinstanceId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", provider.Id)
+			})
 			if err == sql.ErrNoRows {
 				self.DBInstanceId = ""
 			}
@@ -493,7 +500,9 @@ func (manager *SDBInstanceBackupManager) newFromCloudDBInstanceBackup(
 	backup.ExternalId = extBackup.GetGlobalId()
 
 	if dbinstanceId := extBackup.GetDBInstanceId(); len(dbinstanceId) > 0 {
-		_dbinstance, err := db.FetchByExternalId(DBInstanceManager, dbinstanceId)
+		_dbinstance, err := db.FetchByExternalIdAndManagerId(DBInstanceManager, dbinstanceId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+			return q.Equals("manager_id", provider.Id)
+		})
 		if err != nil {
 			log.Warningf("failed to found dbinstance for backup %s by externalId: %s error: %v", backup.Name, dbinstanceId, err)
 		} else {

pkg/compute/models/dbinstance_skus.go

@@ -165,15 +165,20 @@ func (manager *SDBInstanceSkuManager) ListItemFilter(
 		q = q.In("engine_version", query.EngineVersion)
 	}
 
-	zones := ZoneManager.Query().SubQuery()
-	for k, v := range map[string][]string{"zone1": query.Zone1, "zone2": query.Zone2, "zone3": query.Zone3} {
-		if len(v) > 0 {
-			q = q.Join(zones, sqlchemy.Equals(zones.Field("id"), q.Field(k))).Filter(
-				sqlchemy.OR(
-					sqlchemy.In(zones.Field("name"), v),
-					sqlchemy.In(zones.Field("id"), v),
-				),
-			)
+	for k, zoneIds := range map[string][]string{"zone1": query.Zone1, "zone2": query.Zone2, "zone3": query.Zone3} {
+		ids := []string{}
+		for _, zoneId := range zoneIds {
+			zone, err := ZoneManager.FetchByIdOrName(userCred, zoneId)
+			if err != nil {
+				if errors.Cause(err) == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("zone", zoneId)
+				}
+				return nil, httperrors.NewGeneralError(err)
+			}
+			ids = append(ids, zone.GetId())
+		}
+		if len(ids) > 0 {
+			q = q.In(k, ids)
 		}
 	}
 

pkg/compute/models/dbinstancenetworks.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/netutils"
+	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -162,9 +163,15 @@ func (manager *SDBInstanceNetworkManager) SyncDBInstanceNetwork(ctx context.Cont
 
 func (self *SDBInstanceNetwork) syncWithCloudDBNetwork(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *SDBInstance, network *cloudprovider.SDBInstanceNetwork) error {
 	_, err := db.UpdateWithLock(ctx, self, func() error {
-		_localnetwork, err := db.FetchByExternalId(NetworkManager, network.NetworkId)
+		_localnetwork, err := db.FetchByExternalIdAndManagerId(NetworkManager, network.NetworkId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+			wire := WireManager.Query().SubQuery()
+			vpc := VpcManager.Query().SubQuery()
+			return q.Join(wire, sqlchemy.Equals(wire.Field("id"), q.Field("wire_id"))).
+				Join(vpc, sqlchemy.Equals(vpc.Field("id"), wire.Field("vpc_id"))).
+				Filter(sqlchemy.Equals(vpc.Field("manager_id"), dbinstance.ManagerId))
+		})
 		if err != nil {
-			return errors.Wrapf(err, "FetchByExternalId")
+			return errors.Wrapf(err, "FetchByExternalIdAndManagerId")
 		}
 		localnetwork := _localnetwork.(*SNetwork)
 		self.NetworkId = localnetwork.Id
@@ -194,9 +201,15 @@ func (manager *SDBInstanceNetworkManager) newFromCloudDBNetwork(ctx context.Cont
 	dbNetwork.SetModelManager(manager, &dbNetwork)
 
 	dbNetwork.DBInstanceId = dbinstance.Id
-	_localnetwork, err := db.FetchByExternalId(NetworkManager, network.NetworkId)
+	_localnetwork, err := db.FetchByExternalIdAndManagerId(NetworkManager, network.NetworkId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		wire := WireManager.Query().SubQuery()
+		vpc := VpcManager.Query().SubQuery()
+		return q.Join(wire, sqlchemy.Equals(wire.Field("id"), q.Field("wire_id"))).
+			Join(vpc, sqlchemy.Equals(vpc.Field("id"), wire.Field("vpc_id"))).
+			Filter(sqlchemy.Equals(vpc.Field("manager_id"), dbinstance.ManagerId))
+	})
 	if err != nil {
-		return errors.Wrapf(err, "newFromCloudDBNetwork.FetchByExternalId")
+		return errors.Wrapf(err, "newFromCloudDBNetwork.FetchByExternalIdAndManagerId")
 	}
 
 	localnetwork := _localnetwork.(*SNetwork)

pkg/compute/models/dbinstances.go

@@ -675,22 +675,19 @@ func (self *SDBInstance) AllowPerformRecovery(ctx context.Context, userCred mccl
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "recovery")
 }
 
-func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SDBInstanceRecoveryConfigInput) (jsonutils.JSONObject, error) {
 	if !utils.IsInStringArray(self.Status, []string{api.DBINSTANCE_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("Cannot do recovery dbinstance in status %s required status %s", self.Status, api.DBINSTANCE_RUNNING)
 	}
 
-	params := data.(*jsonutils.JSONDict)
-	backupV := validators.NewModelIdOrNameValidator("dbinstancebackup", "dbinstancebackup", userCred)
-	err := backupV.Validate(params)
+	_backup, err := DBInstanceBackupManager.FetchByIdOrName(userCred, input.DBInstancebackupId)
 	if err != nil {
-		return nil, err
-	}
-	input := &api.SDBInstanceRecoveryConfigInput{}
-	err = params.Unmarshal(input)
-	if err != nil {
-		return nil, httperrors.NewInputParameterError("Failed to unmarshal input config: %v", err)
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2("dbinstancebackup", input.DBInstancebackupId)
+		}
+		return nil, httperrors.NewGeneralError(err)
 	}
+	input.DBInstancebackupId = _backup.GetId()
 
 	databases, err := self.GetDBInstanceDatabases()
 	if err != nil {
@@ -702,7 +699,7 @@ func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.
 		dbDatabases = append(dbDatabases, database.Name)
 	}
 
-	backup := backupV.Model.(*SDBInstanceBackup)
+	backup := _backup.(*SDBInstanceBackup)
 	for src, dest := range input.Databases {
 		if len(dest) == 0 {
 			dest = src
@@ -725,6 +722,20 @@ func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.
 		return nil, httperrors.NewInputParameterError("backup and instance not in same cloudregion")
 	}
 
+	if len(backup.Engine) > 0 && backup.Engine != self.Engine {
+		return nil, httperrors.NewInputParameterError("can not recover data from diff rds engine")
+	}
+
+	driver, err := self.GetRegionDriver()
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+
+	err = driver.ValidateDBInstanceRecovery(ctx, userCred, self, backup, input)
+	if err != nil {
+		return nil, err
+	}
+
 	return nil, self.StartDBInstanceRecoveryTask(ctx, userCred, input.JSON(input), "")
 }
 
@@ -1236,12 +1247,16 @@ func (manager *SDBInstanceManager) SyncDBInstanceMasterId(ctx context.Context, u
 	for _, instance := range cloudDBInstances {
 		masterId := instance.GetMasterInstanceId()
 		if len(masterId) > 0 {
-			master, err := db.FetchByExternalId(manager, masterId)
+			master, err := db.FetchByExternalIdAndManagerId(manager, masterId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", provider.Id)
+			})
 			if err != nil {
 				log.Errorf("failed to found master dbinstance by externalId: %s error: %v", masterId, err)
 				continue
 			}
-			slave, err := db.FetchByExternalId(manager, instance.GetGlobalId())
+			slave, err := db.FetchByExternalIdAndManagerId(manager, instance.GetGlobalId(), func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", provider.Id)
+			})
 			if err != nil {
 				log.Errorf("failed to found local dbinstance by externalId %s error: %v", instance.GetGlobalId(), err)
 				continue
@@ -1296,7 +1311,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 			syncResult.UpdateError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+		syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 		localDBInstances = append(localDBInstances, commondb[i])
 		remoteDBInstances = append(remoteDBInstances, commonext[i])
 		syncResult.Update()
@@ -1308,7 +1323,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 			syncResult.AddError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, instance, added[i])
+		syncVirtualResourceMetadata(ctx, userCred, instance, added[i])
 		localDBInstances = append(localDBInstances, *instance)
 		remoteDBInstances = append(remoteDBInstances, added[i])
 		syncResult.Add()
@@ -1320,9 +1335,12 @@ func (self *SDBInstance) syncRemoveCloudDBInstance(ctx context.Context, userCred
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil { // cannot delete
-		return self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.RealDelete(ctx, userCred)
 }
@@ -1492,7 +1510,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if len(self.VpcId) == 0 {
 			if vpcId := extInstance.GetIVpcId(); len(vpcId) > 0 {
-				vpc, err := db.FetchByExternalId(VpcManager, vpcId)
+				vpc, err := db.FetchByExternalIdAndManagerId(VpcManager, vpcId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+					return q.Equals("manager_id", provider.Id)
+				})
 				if err != nil {
 					return errors.Wrapf(err, "SyncWithCloudDBInstance.FetchVpcId")
 				}
@@ -1577,7 +1597,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	}
 
 	if vpcId := extInstance.GetIVpcId(); len(vpcId) > 0 {
-		vpc, err := db.FetchByExternalId(VpcManager, vpcId)
+		vpc, err := db.FetchByExternalIdAndManagerId(VpcManager, vpcId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+			return q.Equals("manager_id", provider.Id)
+		})
 		if err != nil {
 			return nil, errors.Wrapf(err, "newFromCloudDBInstance.FetchVpcId")
 		}

pkg/compute/models/disks.go

@@ -495,7 +495,9 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc
 			input.Hypervisor,
 		)
 	} else {
-		diskConfig.Backend = api.STORAGE_LOCAL
+		if len(diskConfig.Backend) == 0 {
+			diskConfig.Backend = api.STORAGE_LOCAL
+		}
 		serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor)
 		if err != nil {
 			return nil, err
@@ -1138,6 +1140,14 @@ func (self *SDisk) IsLocal() bool {
 	return false
 }
 
+func (self *SDisk) GetCloudproviderId() string {
+	storage := self.GetStorage()
+	if storage != nil {
+		return storage.GetCloudproviderId()
+	}
+	return ""
+}
+
 func (self *SDisk) GetStorage() *SStorage {
 	store, _ := StorageManager.FetchById(self.StorageId)
 	if store != nil {
@@ -1209,18 +1219,26 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro
 	return disks, nil
 }
 
-func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, vdisk cloudprovider.ICloudDisk, index int, syncOwnerId mcclient.IIdentityProvider) (*SDisk, error) {
+func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, vdisk cloudprovider.ICloudDisk, index int, syncOwnerId mcclient.IIdentityProvider, managerId string) (*SDisk, error) {
 	// ownerProjId := projectId
 
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, syncOwnerId))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, syncOwnerId))
 
-	diskObj, err := db.FetchByExternalId(manager, vdisk.GetGlobalId())
+	diskObj, err := db.FetchByExternalIdAndManagerId(manager, vdisk.GetGlobalId(), func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		sq := StorageManager.Query().SubQuery()
+		return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("storage_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), managerId))
+	})
 	if err != nil {
 		if err == sql.ErrNoRows {
-			vstorage, _ := vdisk.GetIStorage()
+			vstorage, err := vdisk.GetIStorage()
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to GetIStorage of vdisk %q", vdisk.GetName())
+			}
 
-			storageObj, err := db.FetchByExternalId(StorageManager, vstorage.GetGlobalId())
+			storageObj, err := db.FetchByExternalIdAndManagerId(StorageManager, vstorage.GetGlobalId(), func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", managerId)
+			})
 			if err != nil {
 				log.Errorf("cannot find storage of vdisk %s", err)
 				return nil, err
@@ -1232,7 +1250,7 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien
 		}
 	} else {
 		disk := diskObj.(*SDisk)
-		err = disk.syncWithCloudDisk(ctx, userCred, provider, vdisk, index, syncOwnerId)
+		err = disk.syncWithCloudDisk(ctx, userCred, provider, vdisk, index, syncOwnerId, managerId)
 		if err != nil {
 			return nil, err
 		}
@@ -1277,11 +1295,11 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 	}
 
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, syncOwnerId)
+		err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, syncOwnerId, storage.ManagerId)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+			syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 			localDisks = append(localDisks, commondb[i])
 			remoteDisks = append(remoteDisks, commonext[i])
 			syncResult.Update()
@@ -1290,7 +1308,10 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 
 	for i := 0; i < len(added); i += 1 {
 		extId := added[i].GetGlobalId()
-		_disk, err := db.FetchByExternalId(manager, extId)
+		_disk, err := db.FetchByExternalIdAndManagerId(manager, extId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+			sq := StorageManager.Query().SubQuery()
+			return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("storage_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), storage.ManagerId))
+		})
 		if err != nil && err != sql.ErrNoRows {
 			//主要是显示duplicate err及 general err,方便排错
 			msg := fmt.Errorf("failed to found disk by external Id %s error: %v", extId, err)
@@ -1299,7 +1320,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 		}
 		if _disk != nil {
 			disk := _disk.(*SDisk)
-			err = disk.syncDiskStorage(ctx, userCred, added[i])
+			err = disk.syncDiskStorage(ctx, userCred, added[i], storage.ManagerId)
 			if err != nil {
 				syncResult.UpdateError(err)
 			} else {
@@ -1311,7 +1332,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, new, added[i])
+			syncVirtualResourceMetadata(ctx, userCred, new, added[i])
 			localDisks = append(localDisks, *new)
 			remoteDisks = append(remoteDisks, added[i])
 			syncResult.Add()
@@ -1321,7 +1342,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To
 	return localDisks, remoteDisks, syncResult
 }
 
-func (self *SDisk) syncDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, idisk cloudprovider.ICloudDisk) error {
+func (self *SDisk) syncDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, idisk cloudprovider.ICloudDisk, managerId string) error {
 	extId := idisk.GetGlobalId()
 	istorage, err := idisk.GetIStorage()
 	if err != nil {
@@ -1329,7 +1350,9 @@ func (self *SDisk) syncDiskStorage(ctx context.Context, userCred mcclient.TokenC
 		return err
 	}
 	storageExtId := istorage.GetGlobalId()
-	storage, err := db.FetchByExternalId(StorageManager, storageExtId)
+	storage, err := db.FetchByExternalIdAndManagerId(StorageManager, storageExtId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		return q.Equals("manager_id", managerId)
+	})
 	if err != nil {
 		log.Errorf("failed to found storage by istorage %s error: %v", storageExtId, err)
 		return err
@@ -1381,7 +1404,12 @@ func (self *SDisk) syncRemoveCloudDisk(ctx context.Context, userCred mcclient.To
 	iDisk, err := iregion.GetIDiskById(self.ExternalId)
 	if err == nil {
 		if storageId := iDisk.GetIStorageId(); len(storageId) > 0 {
-			storage, err := db.FetchByExternalId(StorageManager, storageId)
+			storage, err := db.FetchByExternalIdAndManagerId(StorageManager, storageId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				if s := self.GetStorage(); s != nil {
+					return q.Equals("manager_id", s.ManagerId)
+				}
+				return q
+			})
 			if err == nil {
 				_, err = db.Update(self, func() error {
 					self.StorageId = storage.GetId()
@@ -1407,7 +1435,7 @@ func (self *SDisk) syncRemoveCloudDisk(ctx context.Context, userCred mcclient.To
 	return self.RealDelete(ctx, userCred)
 }
 
-func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, extDisk cloudprovider.ICloudDisk, index int, syncOwnerId mcclient.IIdentityProvider) error {
+func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, extDisk cloudprovider.ICloudDisk, index int, syncOwnerId mcclient.IIdentityProvider, managerId string) error {
 	recycle := false
 	guests := self.GetGuests()
 	if provider.GetFactory().IsSupportPrepaidResources() && len(guests) == 1 && guests[0].IsPrepaidRecycle() {

pkg/compute/models/dynamicschedtags.go

@@ -105,7 +105,7 @@ type SDynamicschedtag struct {
 
 	// 动态调度标间的匹配条件
 	// example: host.sys_load > 1.5 || host.mem_used_percent > 0.7 => "high_load"
-	Condition string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`
+	Condition string `width:"1024" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`
 
 	// 动态调度标签对应的调度标签
 	// SchedtagId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`

pkg/compute/models/elasticcache_instances.go

@@ -472,7 +472,7 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 			syncResult.UpdateError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, &commondb[i], commonext[i])
+		syncVirtualResourceMetadata(ctx, userCred, &commondb[i], commonext[i])
 		localElasticcaches = append(localElasticcaches, commondb[i])
 		remoteElasticcaches = append(remoteElasticcaches, commonext[i])
 		syncResult.Update()
@@ -484,7 +484,7 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 			syncResult.AddError(err)
 			continue
 		}
-		syncMetadata(ctx, userCred, instance, added[i])
+		syncVirtualResourceMetadata(ctx, userCred, instance, added[i])
 		localElasticcaches = append(localElasticcaches, *instance)
 		remoteElasticcaches = append(remoteElasticcaches, added[i])
 		syncResult.Add()
@@ -496,9 +496,12 @@ func (self *SElasticcache) syncRemoveCloudElasticcache(ctx context.Context, user
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.Delete(ctx, userCred)
 }
@@ -559,7 +562,6 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 	instance.Engine = extInstance.GetEngine()
 	instance.EngineVersion = extInstance.GetEngineVersion()
 
-	instance.NetworkType = extInstance.GetNetworkType()
 	instance.PrivateDNS = extInstance.GetPrivateDNS()
 	instance.PrivateIpAddr = extInstance.GetPrivateIpAddr()
 	instance.PrivateConnectPort = extInstance.GetPrivateConnectPort()
@@ -570,28 +572,57 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 	instance.MaintainEndTime = extInstance.GetMaintainEndTime()
 	instance.AuthMode = extInstance.GetAuthMode()
 
+	var zone *SZone
 	if zoneId := extInstance.GetZoneId(); len(zoneId) > 0 {
-		zone, err := db.FetchByExternalId(ZoneManager, zoneId)
+		_zone, err := db.FetchByExternalId(ZoneManager, zoneId)
 		if err != nil {
 			return nil, errors.Wrapf(err, "newFromCloudElasticcache.FetchZoneId")
 		}
-		instance.ZoneId = zone.GetId()
+		instance.ZoneId = _zone.GetId()
+		zone = _zone.(*SZone)
 	}
 
-	if vpcId := extInstance.GetVpcId(); len(vpcId) > 0 {
-		vpc, err := db.FetchByExternalId(VpcManager, vpcId)
+	instance.NetworkType = extInstance.GetNetworkType()
+	if instance.NetworkType == api.LB_NETWORK_TYPE_CLASSIC {
+		vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(provider, region)
 		if err != nil {
-			return nil, errors.Wrapf(err, "newFromCloudElasticcache.FetchVpcId")
+			return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
 		}
 		instance.VpcId = vpc.GetId()
-	}
 
-	if networkId := extInstance.GetNetworkId(); len(networkId) > 0 {
-		network, err := db.FetchByExternalId(NetworkManager, networkId)
+		wire, err := WireManager.GetOrCreateWireForClassicNetwork(vpc, zone)
 		if err != nil {
-			return nil, errors.Wrapf(err, "newFromCloudElasticcache.FetchNetworkId")
+			return nil, errors.Wrap(err, "NewWireForClassicNetwork")
+		}
+		network, err := NetworkManager.GetOrCreateClassicNetwork(wire)
+		if err != nil {
+			return nil, errors.Wrap(err, "GetOrCreateClassicNetwork")
 		}
 		instance.NetworkId = network.GetId()
+	} else {
+		if vpcId := extInstance.GetVpcId(); len(vpcId) > 0 {
+			vpc, err := db.FetchByExternalIdAndManagerId(VpcManager, vpcId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				return q.Equals("manager_id", provider.Id)
+			})
+			if err != nil {
+				return nil, errors.Wrapf(err, "newFromCloudElasticcache.FetchVpcId")
+			}
+			instance.VpcId = vpc.GetId()
+		}
+
+		if networkId := extInstance.GetNetworkId(); len(networkId) > 0 {
+			network, err := db.FetchByExternalIdAndManagerId(NetworkManager, networkId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+				wire := WireManager.Query().SubQuery()
+				vpc := VpcManager.Query().SubQuery()
+				return q.Join(wire, sqlchemy.Equals(wire.Field("id"), q.Field("wire_id"))).
+					Join(vpc, sqlchemy.Equals(vpc.Field("id"), wire.Field("vpc_id"))).
+					Filter(sqlchemy.Equals(vpc.Field("manager_id"), provider.Id))
+			})
+			if err != nil {
+				return nil, errors.Wrapf(err, "newFromCloudElasticcache.FetchNetworkId")
+			}
+			instance.NetworkId = network.GetId()
+		}
 	}
 
 	if createdAt := extInstance.GetCreatedAt(); !createdAt.IsZero() {
@@ -870,7 +901,7 @@ func (self *SElasticcache) GetCreateHuaweiElasticcacheParams(data *jsonutils.JSO
 
 	// fill security group here
 	if len(self.SecurityGroupId) > 0 {
-		sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id)
+		sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id, "")
 		if err != nil {
 			return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.SecurityGroup")
 		}

pkg/compute/models/elasticcache_skus.go

@@ -448,6 +448,8 @@ func (self *SElasticcacheSku) syncWithCloudSku(ctx context.Context, userCred mcc
 	_, err := db.Update(self, func() error {
 		self.PrepaidStatus = extSku.PrepaidStatus
 		self.PostpaidStatus = extSku.PostpaidStatus
+		self.ZoneId = extSku.ZoneId
+		self.SlaveZoneId = extSku.SlaveZoneId
 		return nil
 	})
 	return err