Merge pull request #9115 from yousong/automated-cherry-pick-of-#9109-upstream-release-3.1

Automated cherry pick of #9109: fix(host): make sure it won't match vpc guests by accident

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/logger /opt/yunion/bin/logger
 

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/monitor /opt/yunion/bin/monitor
 

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -7,7 +7,7 @@ ENV TZ Asia/Shanghai
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates && \
+    apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.yunionconf

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/yunionconf /opt/yunion/bin/yunionconf
 

cmd/climc/shell/cloudaccounts.go

@@ -26,6 +26,28 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
+func parseGcpCredential(filename string) (jsonutils.JSONObject, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	authParams, err := jsonutils.Parse(data)
+	if err != nil {
+		return nil, err
+	}
+	ret := jsonutils.NewDict()
+	for _, k := range []string{
+		"client_email",
+		"project_id",
+		"private_key_id",
+		"private_key",
+	} {
+		v, _ := authParams.Get(k)
+		ret.Add(v, fmt.Sprintf("gcp_%s", k))
+	}
+	return ret, nil
+}
+
 func init() {
 
 	type CloudaccountListOptions struct {
@@ -138,11 +160,7 @@ func init() {
 	R(&options.SGoogleCloudAccountCreateOptions{}, "cloud-account-create-google", "Create a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountCreateOptions) error {
 		params := jsonutils.Marshal(args)
 		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Google"), "provider")
-		data, err := ioutil.ReadFile(args.GoogleJsonFile)
-		if err != nil {
-			return err
-		}
-		authParams, err := jsonutils.Parse(data)
+		authParams, err := parseGcpCredential(args.GoogleJsonFile)
 		if err != nil {
 			return err
 		}
@@ -688,11 +706,7 @@ func init() {
 	})
 
 	R(&options.SGoogleCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-google", "Update credential of a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountUpdateCredentialOptions) error {
-		data, err := ioutil.ReadFile(args.GoogleJsonFile)
-		if err != nil {
-			return err
-		}
-		params, err := jsonutils.Parse(data)
+		params, err := parseGcpCredential(args.GoogleJsonFile)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/copyright.go

@@ -15,7 +15,14 @@
 package shell
 
 import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
 
 	"yunion.io/x/jsonutils"
 
@@ -56,4 +63,78 @@ func init() {
 		printObject(r)
 		return nil
 	})
+
+	type EnterpriseUpdateOptions struct {
+		Name      string `help:"Enterprise name"`
+		Copyright string `help:"The Email"`
+		Logo      string `help:"logo file path"`
+		LoginLogo string `help:"login page logo file path"`
+		Favicon   string `help:"favicon path"`
+	}
+
+	R(&EnterpriseUpdateOptions{}, "infos-update", "update enterprise info", func(s *mcclient.ClientSession, args *EnterpriseUpdateOptions) error {
+		var b bytes.Buffer
+		w := multipart.NewWriter(&b)
+		if len(args.Name) > 0 {
+			w.WriteField("name", args.Name)
+		}
+
+		if len(args.Copyright) > 0 {
+			w.WriteField("copyright", args.Copyright)
+		}
+
+		writeFile := func(filedName, filePath string) error {
+			if len(filePath) == 0 {
+				return nil
+			}
+
+			c, err := ioutil.ReadFile(filePath)
+			if err != nil {
+				return err
+			}
+
+			var v string
+			c64 := base64.StdEncoding.EncodeToString(c)
+			ext := filepath.Ext(filePath)
+			switch ext {
+			case ".png":
+				v = fmt.Sprintf("data:image/png;base64,%s", c64)
+			case ".jpg":
+				v = fmt.Sprintf("data:image/jpg;base64,%s", c64)
+			default:
+				return fmt.Errorf("only support png/jpg image")
+			}
+
+			err = w.WriteField(filedName, v)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		if err := writeFile("logo", args.Logo); err != nil {
+			return err
+		}
+
+		if err := writeFile("login_logo", args.LoginLogo); err != nil {
+			return err
+		}
+
+		if err := writeFile("favicon", args.Favicon); err != nil {
+			return err
+		}
+
+		header := make(http.Header)
+		header.Set("content-type", fmt.Sprintf("multipart/form-data; boundary=%s", w.Boundary()))
+		w.Close()
+
+		ret, err := modules.Info.Update(s, header, bufio.NewReader(&b))
+		if err != nil {
+			return err
+		}
+
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/k8s/event.go

@@ -0,0 +1,9 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initEvent() {
+	initK8sNamespaceResource("event", k8s.Events)
+}

cmd/climc/shell/k8s/k8s.go

@@ -16,6 +16,11 @@ package k8s
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+
+	"github.com/ghodss/yaml"
 
 	"yunion.io/x/jsonutils"
 
@@ -57,7 +62,7 @@ func init() {
 	initPVC()
 	initJob()
 	initCronJob()
-
+	initEvent()
 	initRbac()
 
 	initApp()
@@ -217,6 +222,71 @@ func NewK8sNsResourceGetCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cm
 	)
 }
 
+func NewK8sNsResourceGetRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("show-raw"),
+		fmt.Sprintf("Show k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			ret, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			printObjectYAML(ret)
+			return nil
+		},
+	)
+}
+
+func NewK8sResourceEditRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("edit-raw"),
+		fmt.Sprintf("Edit and update k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			rawData, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			yamlBytes := rawData.YAMLString()
+			tempfile, err := ioutil.TempFile("", fmt.Sprintf("k8s-%s-%s*.yaml", cmdN.Kind, args.NAME))
+			if err != nil {
+				return err
+			}
+			defer os.Remove(tempfile.Name())
+			if _, err := tempfile.Write([]byte(yamlBytes)); err != nil {
+				return err
+			}
+			if err := tempfile.Close(); err != nil {
+				return err
+			}
+
+			cmd := exec.Command("vim", tempfile.Name())
+			cmd.Stdin = os.Stdin
+			cmd.Stdout = os.Stdout
+			if err := cmd.Run(); err != nil {
+				return err
+			}
+			content, err := ioutil.ReadFile(tempfile.Name())
+			if err != nil {
+				return err
+			}
+			jsonBytes, err := yaml.YAMLToJSON(content)
+			if err != nil {
+				return err
+			}
+			body, err := jsonutils.Parse(jsonBytes)
+			if err != nil {
+				return err
+			}
+			if _, err := manager.UpdateRaw(s, args.NAME, args.Params(), body.(*jsonutils.JSONDict)); err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+}
+
 func NewK8sResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cmd {
 	return NewCommand(
 		&o.ResourceDeleteOptions{},
@@ -244,11 +314,13 @@ func NewK8sNsResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager)
 	return deleteCmd
 }
 
-func initK8sNamespaceResource(kind string, manager modulebase.Manager) *ShellCommands {
+func initK8sNamespaceResource(kind string, manager k8s.IClusterResourceManager) *ShellCommands {
 	cmdN := NewCmdNameFactory(kind)
 	return NewShellCommands(cmdN.Do).AddR(
 		NewK8sNsResourceListCmd(cmdN, manager),
 		NewK8sNsResourceGetCmd(cmdN, manager),
 		NewK8sNsResourceDeleteCmd(cmdN, manager),
+		NewK8sNsResourceGetRawCmd(cmdN, manager),
+		NewK8sResourceEditRawCmd(cmdN, manager),
 	)
 }

cmd/climc/shell/policies.go

@@ -285,12 +285,30 @@ func init() {
 		if err != nil {
 			log.Fatalf("Set log level %q: %v", "debug", err)
 		}
+		if args.Debug {
+			rbacutils.ShowMatchRuleDebug = true
+		}
 		auth.InitFromClientSession(s)
 		policy.EnableGlobalRbac(15*time.Second, 15*time.Second, false)
 		if args.Debug {
 			consts.EnableRbacDebug()
 		}
 
+		findPolicy := false
+		for !findPolicy {
+			all := policy.PolicyManager.AllPolicies()
+			for _, allP := range all {
+				if len(allP) > 0 {
+					findPolicy = true
+					break
+				}
+			}
+			if findPolicy {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
 		req := jsonutils.NewDict()
 		for i := 0; i < len(args.Request); i += 1 {
 			parts := strings.Split(args.Request[i], ":")
@@ -356,6 +374,7 @@ func init() {
 				Context: mcclient.SAuthContext{
 					Ip: args.Ip,
 				},
+				Token: "faketoken",
 			}
 		} else {
 			token = s.GetToken()
@@ -367,6 +386,10 @@ func init() {
 		}
 		printObject(result)
 
+		for _, r := range args.Role {
+			fmt.Println("role", r, "matched policies:", policy.PolicyManager.RoleMatchPolicies(r))
+		}
+
 		fmt.Println("userCred:", token)
 		for _, scope := range []rbacutils.TRbacScope{
 			rbacutils.ScopeSystem,
@@ -375,7 +398,7 @@ func init() {
 			rbacutils.ScopeUser,
 			rbacutils.ScopeNone,
 		} {
-			m := policy.PolicyManager.MatchedPolicies(scope, token)
+			m := policy.PolicyManager.MatchedPolicyNames(scope, token)
 			fmt.Println("matched", scope, "policies:", m)
 		}
 

cmd/climc/shell/servers.go

@@ -988,6 +988,7 @@ func init() {
 				config.Hosts[i].HostIp = yamlConfig.Hosts[i].HostIp
 				config.Hosts[i].XmlFilePath = yamlConfig.Hosts[i].XmlFilePath
 				config.Hosts[i].Servers = make([]compute.SLibvirtServerConfig, len(yamlConfig.Hosts[i].Servers))
+				config.Hosts[i].MonitorPath = yamlConfig.Hosts[i].MonitorPath
 				for j := 0; j < len(yamlConfig.Hosts[i].Servers); j++ {
 					config.Hosts[i].Servers[j].MacIp = make(map[string]string)
 					mac := yamlConfig.Hosts[i].Servers[j].Mac
@@ -1010,11 +1011,11 @@ func init() {
 		if err != nil {
 			return err
 		}
-		for i := 0; i < len(params); i++ {
-			val := jsonutils.NewDict()
-			val.Set(modules.Servers.KeywordPlural, params[i])
-			params[i] = val
-		}
+		//for i := 0; i < len(params); i++ {
+		//	val := jsonutils.NewDict()
+		//	val.Set(modules.Servers.KeywordPlural, params[i])
+		//	params[i] = val
+		//}
 
 		results := modules.Servers.BatchPerformClassAction(s, "import-from-libvirt", params)
 		printBatchResults(results, modules.Servers.GetColumns(s))

go.mod

@@ -3,7 +3,6 @@ module yunion.io/x/onecloud
 go 1.12
 
 require (
-	cloud.google.com/go v0.38.0
 	github.com/360EntSecGroup-Skylar/excelize v1.4.0
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible
 	github.com/Azure/go-autorest/autorest v0.9.3
@@ -115,7 +114,6 @@ require (
 	golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
-	google.golang.org/api v0.13.0 // indirect
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
 	google.golang.org/grpc v1.23.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
@@ -129,12 +127,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
-	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
+	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
+	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
+	yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
-	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
+	yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e
 )
 
 replace (

go.sum

@@ -371,8 +371,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
@@ -897,8 +895,6 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmK
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
-google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
@@ -1015,19 +1011,21 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda h1:wSwQj3MDGchGYM2RAo1riYhrTz4apH+5XyBnchuDi84=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501 h1:i1r9XvbdxH3FgTCLmTaRi3MzQqhiQimXJRlUOPgrxnU=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa h1:VizPfW8+mLFEE7W/97zxQJbfdxchi2dn1M/Y7YBwTTc=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e h1:pctCe/EPel3F1B83pJ2q9b34Umd1NdbLW1Yd+Lzur2s=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -361,7 +361,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
+		httperrors.ForbiddenError(w, "user forbidden login from web")
 		return false
 	}
 	return true
@@ -570,6 +570,7 @@ func (this *projectRoles) add(roleId, roleName string) {
 
 func (this *projectRoles) getToken(scope rbacutils.TRbacScope, user, userId, domain, domainId string, ip string) mcclient.TokenCredential {
 	return &mcclient.SSimpleToken{
+		Token:           "faketoken",
 		Domain:          domain,
 		DomainId:        domainId,
 		User:            user,
@@ -609,12 +610,12 @@ func (this *projectRoles) json(user, userId, domain, domainId string, ip string)
 	}
 	obj.Add(roles, "roles")
 	for _, scope := range []rbacutils.TRbacScope{
-		rbacutils.ScopeSystem,
-		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeSystem,
 	} {
 		token := this.getToken(scope, user, userId, domain, domainId, ip)
-		matches := policy.PolicyManager.MatchedPolicies(scope, token)
+		matches := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		obj.Add(jsonutils.NewStringArray(matches), fmt.Sprintf("%s_policies", scope))
 		if len(matches) > 0 {
 			obj.Add(jsonutils.JSONTrue, fmt.Sprintf("%s_capable", scope))
@@ -741,7 +742,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
 	} {
-		p := policy.PolicyManager.MatchedPolicies(scope, token)
+		p := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		data.Add(jsonutils.NewStringArray(p), fmt.Sprintf("%s_policies", scope))
 		if scope == rbacutils.ScopeSystem {
 			data.Add(jsonutils.NewStringArray(p), "admin_policies")

pkg/apigateway/handler/init.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
+)

pkg/apigateway/handler/misc.go

@@ -28,6 +28,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -171,19 +173,23 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	}
 
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +205,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 

pkg/apigateway/handler/resource.go

@@ -606,28 +606,45 @@ func (f *ResourceHandlers) patchJointHandler(ctx context.Context, w http.Respons
 	}
 }
 
-// batch update Joint
+// * batch update Joint
+// * put specific
+// /<resname>/<resid>/<resname2>
+// /<resname>/<resid>/<spec>
 func (f *ResourceHandlers) batchUpdateJointHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	req := newRequest(ctx, w, r).WithMod1().WithMod2()
+	req := newRequest(ctx, w, r).WithMod1()
 	if err := req.Error(); err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
 	}
 	session := req.Session()
 	module := req.Mod1()
-	module2 := req.Mod2()
 	body := req.Body()
 	query := req.Query()
 
-	idlist := fetchIdList(query, w)
-	if idlist == nil {
+	if idlist, _ := query.GetArray("id"); len(idlist) == 0 {
+		// do put specific
+		spec := req.ResName2()
+		obj, e := module.PutSpecific(session, req.ResID(), spec, query, body)
+		if e != nil {
+			httperrors.GeneralServerError(w, e)
+		} else {
+			appsrv.SendJSON(w, obj)
+		}
 		return
 	}
+
+	req = req.WithMod2()
+	if err := req.Error(); err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	module2 := req.Mod2()
 	jmod, e := modulebase.GetJointModule2(session, module, module2)
 	if e != nil { // update joint
 		httperrors.GeneralServerError(w, e)
 		return
 	}
+	idlist := fetchIdList(query, w)
 	ret := jmod.BatchUpdate(session, req.ResID(), idlist, query, body)
 	w.WriteHeader(207)
 	appsrv.SendJSON(w, modulebase.SubmitResults2JSON(ret))

pkg/apis/cloudcommon/proxy/proxy.go

@@ -39,3 +39,17 @@ func (ps *SProxySetting) IsZero() bool {
 	}
 	return false
 }
+
+type ProxySettingResourceInput struct {
+	// 代理配置
+	ProxySetting string `json:"proxy_setting"`
+
+	// swagger:ignore
+	// Deprecated
+	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
+}
+
+type ProxySettingTestInput struct {
+	HttpProxy  string
+	HttpsProxy string
+}

pkg/apis/cloudevent/cloudevent.go

@@ -18,13 +18,59 @@ import (
 	"time"
 
 	"yunion.io/x/onecloud/pkg/apis"
-	"yunion.io/x/onecloud/pkg/apis/compute"
 )
 
 type CloudeventListInput struct {
-	apis.VirtualResourceListInput
+	apis.ModelBaseListInput
 
-	compute.ManagedResourceListInput
+	// 列出指定云平台的资源，支持的云平台如下
+	//
+	// | Provider  | 开始支持版本 | 平台                                |
+	// |-----------|------------|-------------------------------------|
+	// | OneCloud  | 0.0        | OneCloud内置私有云，包括KVM和裸金属管理 |
+	// | VMware    | 1.2        | VMware vCenter                      |
+	// | OpenStack | 2.6        | OpenStack M版本以上私有云             |
+	// | ZStack    | 2.10       | ZStack私有云                         |
+	// | Aliyun    | 2.0        | 阿里云                               |
+	// | Aws       | 2.3        | Amazon AWS                          |
+	// | Azure     | 2.2        | Microsoft Azure                     |
+	// | Google    | 2.13       | Google Cloud Platform               |
+	// | Qcloud    | 2.3        | 腾讯云                               |
+	// | Huawei    | 2.5        | 华为公有云                           |
+	// | Ucloud    | 2.7        | UCLOUD                               |
+	// | Ctyun     | 2.13       | 天翼云                               |
+	// | S3        | 2.11       | 通用s3对象存储                        |
+	// | Ceph      | 2.11       | Ceph对象存储                         |
+	// | Xsky      | 2.11       | XSKY启明星辰Ceph对象存储              |
+	//
+	// enum: OneCloud,VMware,Aliyun,Qcloud,Azure,Aws,Huawei,OpenStack,Ucloud,ZStack,Google,Ctyun,S3,Ceph,Xsky"
+	Providers []string `json:"providers"`
+	// swagger:ignore
+	// Deprecated
+	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
+
+	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
+	//
+	// |   Brand  | Provider | 说明        |
+	// |----------|----------|------------|
+	// | DStack   | ZStack   | 滴滴云私有云 |
+	//
+	Brands []string `json:"brands"`
+	// swagger:ignore
+	// Deprecated
+	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
+
+	// 服务类型
+	Service []string `json:"service"`
+
+	// 订阅
+	Manager []string `json:"manager"`
+
+	// 账号
+	Account []string `json:"account"`
+
+	// 操作类型
+	Action []string `json:"action"`
 
 	// 操作日志起始时间
 	Since time.Time `json:"since"`

pkg/apis/compute/api.go

@@ -285,7 +285,7 @@ type ServerConfigs struct {
 	// swagger:ignore
 	// Deprecated
 	// alias for InstanceType
-	Sku string `json:"sku" deprecated-by:"instance_type"`
+	Sku string `json:"sku" "yunion:deprecated-by":"instance_type"`
 
 	// 虚拟机高可用(创建备机)
 	// default: false
@@ -357,12 +357,13 @@ type ServerCreateInput struct {
 	// required: false
 	UserData string `json:"user_data"`
 
+	// Deprecated
 	// swagger:ignore
-	Keypair string `json:"keypair" deprecated-by:"keypair_id"`
+	KeypairId string `json:"keypair_id" "yunion:deprecated-by":"keypair"`
 
 	// 秘钥对Id
 	// required: false
-	KeypairId string `json:"keypair_id"`
+	Keypair string `json:"keypair"`
 
 	// 密码
 	// 要求: 密码长度 >= 20, 至少包含一个数字一个小写字母一个大小字母及特殊字符~`!@#$%^&*()-_=+[]{}|:';\",./<>?中的一个

pkg/apis/compute/cloudaccount.go

@@ -80,8 +80,7 @@ type CloudaccountCreateInput struct {
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONObject `json:"options"`
 
-	// 代理配置
-	ProxySettingId string `json:"proxy_setting_id"`
+	proxyapi.ProxySettingResourceInput
 
 	cloudprovider.SCloudaccount
 	cloudprovider.SCloudaccountCredential

pkg/apis/compute/cloudprovider.go

@@ -147,30 +147,30 @@ type ManagedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Manager string `json:"manager" deprecated-by:"cloudprovider"`
+	Manager string `json:"manager" "yunion:deprecated-by":"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	ManagerId string `json:"manager_id" deprecated-by:"cloudprovider"`
+	ManagerId string `json:"manager_id" "yunion:deprecated-by":"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudproviderId string `json:"cloudprovider_id" deprecated-by:"cloudprovider"`
+	CloudproviderId string `json:"cloudprovider_id" "yunion:deprecated-by":"cloudprovider"`
 
 	// 列出关联指定云账号(ID或Name)的资源
 	Cloudaccount string `json:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudaccountId string `json:"cloudaccount_id" deprecated-by:"cloudaccount"`
+	CloudaccountId string `json:"cloudaccount_id" "yunion:deprecated-by":"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Account string `json:"account" deprecated-by:"cloudaccount"`
+	Account string `json:"account" "yunion:deprecated-by":"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	AccountId string `json:"account_id" deprecated-by:"cloudaccount"`
+	AccountId string `json:"account_id" "yunion:deprecated-by":"cloudaccount"`
 
 	// 列出指定云平台的资源，支持的云平台如下
 	//
@@ -196,7 +196,7 @@ type ManagedResourceListInput struct {
 	Providers []string `json:"providers"`
 	// swagger:ignore
 	// Deprecated
-	Provider []string `json:"provider" deprecated-by:"providers"`
+	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
 
 	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
 	//
@@ -207,7 +207,7 @@ type ManagedResourceListInput struct {
 	Brands []string `json:"brands"`
 	// swagger:ignore
 	// Deprecated
-	Brand []string `json:"brand" deprecated-by:"brands"`
+	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
 
 	// 列出指定云环境的资源，支持云环境如下：
 	//

pkg/apis/compute/cloudregions_const.go

@@ -51,11 +51,12 @@ const (
 	CITY_FU_ZHOU       = "Fuzhou"      //福州
 	CITY_WU_HAN        = "Wuhan"       //武汉
 	CITY_CHANG_SHA     = "Changsha"    //长沙
-	CITY_SHU_ZHOU      = "Shuzhou"     //苏州
+	CITY_SU_ZHOU       = "Suzhou"      //苏州
 	CITY_BAO_DING      = "Baoding"     //保定
 	CITY_NAN_JING      = "Nanjing"     //南京
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
+	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京
@@ -111,6 +112,7 @@ const (
 	CITY_US_GOV_WEST    = "us-gov-west"    //???
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
+	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -149,6 +151,12 @@ const (
 	// 巴西
 	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
 
+	// 智利
+	CITY_SANTIAGO = "Santiago" // 圣地亚哥
+
+	// 墨西哥
+	CITY_MEXICO = "Mexico" // 墨西哥
+
 	// 荷兰
 	CITY_HOLLAND = "Holland" //荷兰
 
@@ -194,4 +202,6 @@ const (
 	COUNTRY_CODE_VN = "VN" //越南
 	COUNTRY_CODE_CH = "CH" //瑞士
 	COUNTRY_CODE_NO = "NO" //挪威
+	COUNTRY_CODE_MX = "MX" //墨西哥
+	COUNTRY_CODE_CL = "CL" //智利
 )

pkg/apis/compute/dbinstance.go

@@ -212,7 +212,7 @@ type DbinstanceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by dbinstance_id
-	DbinstanceId string `json:"dbinstance_id" deprecated-by:"dbinstance"`
+	DbinstanceId string `json:"dbinstance_id" "yunion:deprecated-by":"dbinstance"`
 }
 
 type DBInstanceDetails struct {

pkg/apis/compute/disk.go

@@ -89,7 +89,7 @@ type SnapshotPolicyFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter disk by snapshotpolicy_id
-	SnapshotpolicyId string `json:"snapshotpolicy_id" deprecated-by:"snapshotpolicy"`
+	SnapshotpolicyId string `json:"snapshotpolicy_id" "yunion:deprecated-by":"snapshotpolicy"`
 }
 
 type DiskListInput struct {
@@ -109,7 +109,7 @@ type DiskListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk type
-	Type string `json:"type" deprecated-by:"disk_type"`
+	Type string `json:"type" "yunion:deprecated-by":"disk_type"`
 	// 过滤指定disk_type的磁盘列表，可能的值为：sys, data, swap. volume
 	//
 	// | disk_type值 | 说明 |
@@ -128,7 +128,7 @@ type DiskFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk_id
-	DiskId string `json:"disk_id" deprecated-by:"disk"`
+	DiskId string `json:"disk_id" "yunion:deprecated-by":"disk"`
 }
 
 type SimpleGuest struct {
@@ -138,8 +138,10 @@ type SimpleGuest struct {
 }
 
 type SimpleSnapshotPolicy struct {
-	RepeatWeekdays []int `json:"repeat_weekdays"`
-	TimePoints     []int `json:"time_points"`
+	Id             string `json:"id"`
+	Name           string `json:"name"`
+	RepeatWeekdays []int  `json:"repeat_weekdays"`
+	TimePoints     []int  `json:"time_points"`
 }
 
 type DiskDetails struct {

pkg/apis/compute/geo_input.go

@@ -27,15 +27,15 @@ type RegionalFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudregionId string `json:"cloudregion_id" deprecated-by:"cloudregion"`
+	CloudregionId string `json:"cloudregion_id" "yunion:deprecated-by":"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Region string `json:"region" deprecated-by:"cloudregion"`
+	Region string `json:"region" "yunion:deprecated-by":"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	RegionId string `json:"region_id" deprecated-by:"cloudregion"`
+	RegionId string `json:"region_id" "yunion:deprecated-by":"cloudregion"`
 }
 
 type ZonalFilterListInput struct {
@@ -46,7 +46,7 @@ type ZonalFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by zone_id
-	ZoneId string `json:"zone_id" deprecated-by:"zone"`
+	ZoneId string `json:"zone_id" "yunion:deprecated-by":"zone"`
 	// 过滤处于多个指定可用区内的资源
 	Zones []string `json:"zones"`
 }
@@ -67,7 +67,7 @@ type HostFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by host_id
-	HostId string `json:"host_id" deprecated-by:"host"`
+	HostId string `json:"host_id" "yunion:deprecated-by":"host"`
 }
 
 type CloudregionListInput struct {

pkg/apis/compute/guests.go

@@ -28,15 +28,15 @@ type ServerFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	ServerId string `json:"server_id" deprecated-by:"server"`
+	ServerId string `json:"server_id" "yunion:deprecated-by":"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	Guest string `json:"guest" deprecated-by:"server"`
+	Guest string `json:"guest" "yunion:deprecated-by":"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	GuestId string `json:"guest_id" deprecated-by:"server"`
+	GuestId string `json:"guest_id" "yunion:deprecated-by":"server"`
 }
 
 type ServerListInput struct {

pkg/apis/compute/instance_group.go

@@ -41,5 +41,5 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// deprecated: true
 	// Filter by instance group Id
-	GroupId string `json:"group_id" deprecated-by:"group"`
+	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
 }

pkg/apis/compute/network.go

@@ -24,7 +24,7 @@ type VpcFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by vpc Id
-	VpcId string `json:"vpc_id" deprecated-by:"vpc"`
+	VpcId string `json:"vpc_id" "yunion:deprecated-by":"vpc"`
 }
 
 type WireFilterListInput struct {
@@ -35,7 +35,7 @@ type WireFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// fitler by wire id
-	WireId string `json:"wire_id" deprecated-by:"wire"`
+	WireId string `json:"wire_id" "yunion:deprecated-by":"wire"`
 }
 
 type NetworkFilterListInput struct {
@@ -46,7 +46,7 @@ type NetworkFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by networkId
-	NetworkId string `json:"network_id" deprecated-by:"network"`
+	NetworkId string `json:"network_id" "yunion:deprecated-by":"network"`
 }
 
 type NetworkListInput struct {
@@ -60,9 +60,13 @@ type NetworkListInput struct {
 
 	WireFilterListInput
 
-	// description: search ip address in network.
+	// description: Exact matching ip address in network.
 	// example: 10.168.222.1
 	Ip string `json:"ip"`
+
+	// description: Fuzzy matching ip address in network.
+	// example: 10.168.222.1
+	IpMatch string `json:"ip_match"`
 }
 
 type NetworkCreateInput struct {

pkg/apis/compute/schedtag.go

@@ -45,7 +45,7 @@ type SchedtagFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by schedtag_id
-	SchedtagId string `json:"schedtag_id" deprecated-by:"schedtag"`
+	SchedtagId string `json:"schedtag_id" "yunion:deprecated-by":"schedtag"`
 }
 
 type SchedtagListInput struct {
@@ -56,7 +56,7 @@ type SchedtagListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by type, alias for resource_type
-	Type string `json:"type" deprecated-by:"resource_type"`
+	Type string `json:"type" "yunion:deprecated-by":"resource_type"`
 }
 
 type SchedtagDetails struct {

pkg/apis/compute/secgroup.go

@@ -162,7 +162,7 @@ type SecgroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by secgroup_id
-	SecgroupId string `json:"secgroup_id" deprecated-by:"secgroup"`
+	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
 }
 
 type SecgroupDetails struct {

pkg/apis/compute/storage_const.go

@@ -144,7 +144,7 @@ type StorageFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by storage_id
-	StorageId string `json:"storage_id" deprecated-by:"storage"`
+	StorageId string `json:"storage_id" "yunion:deprecated-by":"storage"`
 }
 
 type StorageShareFilterListInput struct {

pkg/apis/compute/wire.go

@@ -53,7 +53,7 @@ type WireDetails struct {
 	// example: 1
 	Networks int `json:"networks"`
 	// VPC名称
-	Vpc string `json:"vpc'`
+	Vpc string `json:"vpc"`
 	// VPC外部Id
 	VpcExtId string `json:"vpc_ext_id"`
 

pkg/apis/identity/assignments.go

@@ -62,3 +62,7 @@ func (ra *SRoleAssignment) GetRoles() []string {
 func (ra *SRoleAssignment) GetLoginIp() string {
 	return ""
 }
+
+func (ra *SRoleAssignment) GetTokenString() string {
+	return "faketoken"
+}

pkg/apis/identity/cas.go

@@ -14,8 +14,20 @@
 
 package identity
 
+import "yunion.io/x/pkg/tristate"
+
 type SCASIdpConfigOptions struct {
 	// https://cas.example.org/cas/
 	CASServerURL string `json:"cas_server_url"`
 	Service      string `json:"service"`
+
+	CasProjectAttribute string `json:"cas_project_attribute"`
+
+	AutoCreateCasProject tristate.TriState `json:"auto_create_cas_project"`
+
+	DefaultCasProjectId string `json:"default_cas_project_id"`
+
+	CasRoleAttribute string `json:"cas_role_attribute"`
+
+	DefaultCasRoleId string `json:"default_cas_role_id"`
 }

pkg/apis/identity/config.go

@@ -93,3 +93,19 @@ var (
 		IdpTemplateOpenLDAPSingleDomain: IdentityDriverLDAP,
 	}
 )
+
+type PerformConfigInput struct {
+	// 更新配置的方式
+	// example: update
+	//
+	// | action  |  含义                                         |
+	// |---------|-----------------------------------------------|
+	// | update  | 增量更新配置                                  |
+	// | remove  | 删除指定配置                                  |
+	// | replace | 全量替换配置，如果action为空，则默认为replace |
+	//
+	Action string `json:"action"`
+
+	// 配置信息
+	Config TConfigs `json:"config"`
+}

pkg/apis/identity/consts.go

@@ -144,6 +144,10 @@ var (
 			"etcd_cacert",
 			"etcd_cert",
 			"etcd_key",
+
+			"bootstrap_admin_user_password",
+			"reset_admin_user_password",
+			"fernet_key_repository",
 		},
 	}
 )

pkg/apis/identity/identityprovider.go

@@ -29,3 +29,28 @@ type IdentityProviderDetails struct {
 	ProjectCount int `json:"project_count,allowempty"`
 	GroupCount   int `json:"group_count,allowempty"`
 }
+
+type IdentityProviderCreateInput struct {
+	apis.EnabledStatusStandaloneResourceCreateInput
+
+	// 后端驱动名称
+	Driver string `json:"driver"`
+
+	// 模板名称
+	Template string `json:"template"`
+
+	// 默认导入用户和组的域
+	TargetDomain string `json:"target_domain"`
+	// swagger:ignore
+	// Deprecated
+	TargetDomainId string `json:"target_domain_id" "yunion:deprecated-by":"target_domain"`
+
+	// 新建域的时候是否自动新建第一个项目
+	AutoCreateProject *bool `json:"auto_create_project"`
+
+	// 自动同步间隔，单位：秒
+	SyncIntervalSeconds *int `json:"sync_interval_seconds"`
+
+	// 配置信息
+	Config TConfigs `json:"config"`
+}

pkg/apis/identity/input.go

@@ -48,15 +48,15 @@ type ProjectFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by project_id
-	ProjectId string `json:"project_id" deprecated-by:"project"`
+	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant
-	Tenant string `json:"tenant" deprecated-by:"project"`
+	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant_id
-	TenantId string `json:"tenant_id" deprecated-by:"project"`
+	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
 }
 
 type UserFilterListInput struct {
@@ -65,7 +65,7 @@ type UserFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by user_id
-	UserId string `json:"user_id" deprecated-by:"user"`
+	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
 }
 
 type GroupFilterListInput struct {
@@ -74,7 +74,7 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by group_id
-	GroupId string `json:"group_id" deprecated-by:"group"`
+	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
 }
 
 type RoleFilterListInput struct {
@@ -83,7 +83,7 @@ type RoleFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by role_id
-	RoleId string `json:"role_id" deprecated-by:"role"`
+	RoleId string `json:"role_id" "yunion:deprecated-by":"role"`
 }
 
 type ServiceFilterListInput struct {
@@ -92,7 +92,7 @@ type ServiceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by service_id
-	ServiceId string `json:"service_id" deprecated-by:"service"`
+	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
 }
 
 type RoleListInput struct {

pkg/apis/input.go

@@ -28,11 +28,11 @@ type DomainizedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Project domain Id filter, alias for project_domain
-	ProjectDomainId string `json:"project_domain_id" deprecated-by:"project_domain"`
+	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Domain Id filter, alias for project_domain
-	DomainId string `json:"domain_id" deprecated-by:"project_domain"`
+	DomainId string `json:"domain_id" "yunion:deprecated-by":"project_domain"`
 }
 
 type DomainizedResourceCreateInput struct {
@@ -53,15 +53,15 @@ type ProjectizedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by project_id, alias for project
-	ProjectId string `json:"project_id" deprecated-by:"project"`
+	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant ID or Name, alias for project
-	Tenant string `json:"tenant" deprecated-by:"project"`
+	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant_id, alias for project
-	TenantId string `json:"tenant_id" deprecated-by:"project"`
+	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
 }
 
 type ProjectizedResourceCreateInput struct {
@@ -82,7 +82,7 @@ type UserResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by userId
-	UserId string `json:"user_id" deprecated-by:"user"`
+	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
 }
 
 func (input UserResourceListInput) UserStr() string {

pkg/appsrv/ring.go

@@ -25,10 +25,12 @@ type Ring struct {
 }
 
 func NewRing(size int) *Ring {
-	r := Ring{buffer: make([]interface{}, size+1),
+	r := Ring{
+		buffer: make([]interface{}, size+1),
 		header: 0,
 		tail:   0,
-		lock:   &sync.Mutex{}}
+		lock:   &sync.Mutex{},
+	}
 	return &r
 }
 
@@ -58,6 +60,7 @@ func (r *Ring) Pop() interface{} {
 		return nil
 	}
 	ret := r.buffer[r.tail]
+	r.buffer[r.tail] = nil
 	r.tail = nextPointer(r.tail, len(r.buffer))
 	return ret
 }

pkg/appsrv/ring_test.go

@@ -19,28 +19,33 @@ import (
 )
 
 func TestRing(t *testing.T) {
-	r := NewRing(10)
-	var v int32 = 10
-	r.Push(v)
-	v = 20
-	r.Push(v)
-	v = 30
-	r.Push(v)
-	v1 := r.Pop().(int32)
-	if v1 != 10 {
-		t.Error("Fail")
-	}
-	v2 := r.Pop().(int32)
-	if v2 != 20 {
-		t.Error("Fail")
-	}
-	v3 := r.Pop().(int32)
-	if v3 != 30 {
-		t.Error("Fail")
-	}
-	v4 := r.Pop()
-	if v4 != nil {
-		t.Error("Fail")
+	var (
+		r    = NewRing(10)
+		push = func(v int32) {
+			r.Push(v)
+		}
+		pop = func(want int32) {
+			got := r.Pop().(int32)
+			if got != want {
+				t.Fatalf("got %d, want %d", got, want)
+			}
+			for i := r.header; i != r.tail; i = nextPointer(i, len(r.buffer)) {
+				if r.buffer[i] != nil {
+					t.Fatalf("head %d, tail %d, index %d not nil",
+						r.header, r.tail, i)
+				}
+			}
+		}
+	)
+	push(10)
+	push(20)
+	push(30)
+
+	pop(10)
+	pop(20)
+	pop(30)
+	if v := r.Pop(); v != nil {
+		t.Fatalf("want nil, got %#v", v)
 	}
 }
 

pkg/appsrv/workers.go

@@ -83,9 +83,6 @@ func (worker *SWorker) run() {
 			task := req.(*sWorkerTask)
 			if task.worker != nil {
 				task.worker <- worker
-				// worker channel is buffered
-				// close the worker channel
-				close(task.worker)
 			}
 			execCallback(task)
 		} else {

pkg/baremetal/cronjobs.go

@@ -189,10 +189,11 @@ func (job *SLogFetchJob) Do(ctx context.Context, now time.Time) error {
 	if err != nil {
 		return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_SYSTEM")
 	}
-	err = fetchLogs(job.baremetal, ctx, redfish.EVENT_TYPE_MANAGER)
-	if err != nil {
-		return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_MANAGER")
-	}
+	// no longer fetch management logs
+	// err = fetchLogs(job.baremetal, ctx, redfish.EVENT_TYPE_MANAGER)
+	// if err != nil {
+	// 	return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_MANAGER")
+	// }
 	job.lastTime = now
 	return nil
 }

pkg/baremetal/handler/handlers.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/baremetal"
@@ -71,6 +72,7 @@ func initBaremetalsHandler(app *appsrv.Application) {
 	AddHandler(app, "POST", bmActionPrefix("reset-bmc"), bmObjMiddleware(handleBaremetalResetBMC))
 	AddHandler(app, "POST", bmActionPrefix("ipmi-probe"), bmObjMiddleware(handleBaremetalIpmiProbe))
 	AddHandler(app, "POST", bmActionPrefix("cdrom"), bmObjMiddleware(handleBaremetalCdromTask))
+	AddHandler(app, "POST", bmActionPrefix("jnlp"), bmObjMiddleware(handleBaremetalJnlpTask))
 
 	// server actions handler
 	AddHandler(app, "POST", srvActionPrefix("create"), srvClassMiddleware(handleServerCreate))
@@ -160,6 +162,17 @@ func handleBaremetalCdromTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
 	ctx.ResponseOk()
 }
 
+func handleBaremetalJnlpTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
+	jnlp, err := bm.GetConsoleJNLP(ctx)
+	if err != nil {
+		ctx.ResponseError(errors.Wrap(err, "GetConsoleJNLP"))
+		return
+	}
+	result := jsonutils.NewDict()
+	result.Add(jsonutils.NewString(jnlp), "jnlp")
+	ctx.ResponseJson(result)
+}
+
 func handleServerCreate(ctx *Context, bm *baremetal.SBaremetalInstance) {
 	err := bm.StartServerCreateTask(ctx.UserCred(), ctx.TaskId(), ctx.Data())
 	if err != nil {

pkg/baremetal/manager.go

@@ -64,6 +64,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
+	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
@@ -1235,7 +1236,12 @@ func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nic
 
 func (b *SBaremetalInstance) GetIPMIConfig() *types.SIPMIInfo {
 	conf := b.GetRawIPMIConfig()
-	if conf == nil || conf.Password == "" {
+	if conf == nil {
+		log.Debugf("GetIPMIConfig conf is nil")
+		return nil
+	}
+	if conf.Password == "" {
+		log.Debugf("GetIPMIConfig password is nil")
 		return nil
 	}
 	if conf.Username == "" {
@@ -1254,6 +1260,7 @@ func (b *SBaremetalInstance) GetIPMIConfig() *types.SIPMIInfo {
 	}
 	conf.Password = utils.Unquote(conf.Password) // XXX: remove quotes!!!
 	if conf.IpAddr == "" {
+		log.Debugf("GetIPMIConfig ipaddr s nil")
 		return nil
 	}
 	return conf
@@ -1336,6 +1343,7 @@ func (b *SBaremetalInstance) SetExistingIPMIIPAddr(ipAddr string) {
 func (b *SBaremetalInstance) GetIPMITool() *ipmitool.LanPlusIPMI {
 	conf := b.GetIPMIConfig()
 	if conf == nil {
+		log.Debugf("GetIPMIConfig is nil")
 		return nil
 	}
 	return ipmitool.NewLanPlusIPMI(conf.IpAddr, conf.Username, conf.Password)
@@ -1632,6 +1640,10 @@ func (b *SBaremetalInstance) DelayedSyncIPMIInfo(data jsonutils.JSONObject) (jso
 }
 
 func (b *SBaremetalInstance) DelayedSyncDesc(data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if data == nil {
+		session := b.manager.GetClientSession()
+		data, _ = b.manager.fetchBaremetal(session, b.GetId())
+	}
 	err := b.SaveDesc(data)
 	return nil, err
 }
@@ -1924,6 +1936,25 @@ func (b *SBaremetalInstance) fetchPowerThermalMetrics(ctx context.Context) ([]in
 	return powerMetrics, thermalMetrics, nil
 }
 
+func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error) {
+	cli := b.GetRedfishCli(ctx)
+	if cli != nil {
+		return cli.GetConsoleJNLP(ctx)
+	}
+	conf := b.GetIPMIConfig()
+	bmc := bmconsole.NewBMCConsole(conf.IpAddr, conf.Username, conf.Password, false)
+	manufacture := b.GetManufacture()
+	switch strings.ToLower(manufacture) {
+	case "hp", "hpe":
+		return bmc.GetIloConsoleJNLP(ctx)
+	case "dell", "dell inc.":
+		return bmc.GetIdracConsoleJNLP(ctx, "", "")
+	case "supermicro":
+		return bmc.GetSupermicroConsoleJNLP(ctx)
+	}
+	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
+}
+
 func (b *SBaremetalInstance) getTags() []influxdb.SKeyValue {
 	tags := []influxdb.SKeyValue{
 		{

pkg/baremetal/profiles/profiles.go

@@ -43,7 +43,7 @@ func InspurProfile() IPMIProfile {
 
 func LenovoProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{8},
+		LanChannel: []int{1, 8},
 		RootName:   "root",
 		RootId:     2,
 	}
@@ -51,7 +51,7 @@ func LenovoProfile() IPMIProfile {
 
 func HpProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{2},
+		LanChannel: []int{1, 2},
 		RootName:   "root",
 		RootId:     1,
 	}

pkg/baremetal/service/service.go

@@ -64,6 +64,7 @@ func (s *BaremetalService) StartService() {
 
 	fsdriver.Init(nil)
 	app := app_common.InitApp(&o.Options.BaseOptions, false)
+
 	handler.InitHandlers(app)
 
 	s.startAgent(app)

pkg/baremetal/tasks/base.go

@@ -306,7 +306,13 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 	if err != nil {
 		return errors.Wrapf(err, "Get power status")
 	}
+	maxTries := 10
+	count := 0
 	for status == "" || status == types.POWER_STATUS_OFF {
+		if count > maxTries {
+			break
+		}
+		log.Infof("Try power on %d times, pxe boot %v", count+1, self.PxeBoot)
 		if status == types.POWER_STATUS_OFF {
 			if self.PxeBoot {
 				err = self.Baremetal.DoPXEBoot()
@@ -314,7 +320,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				err = self.Baremetal.DoRedfishPowerOn()
 			}
 			if err != nil {
-				return errors.Wrapf(err, "Do boot power on")
+				log.Warningf("Do boot power on error: %v", err)
 			}
 		}
 		status, err = self.Baremetal.GetPowerStatus()
@@ -328,6 +334,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				return err
 			}
 		}
+		count++
 	}
 	if status != types.POWER_STATUS_ON {
 		return fmt.Errorf("Baremetal invalid restart status: %s", status)

pkg/baremetal/tasks/baseprepare.go

@@ -108,7 +108,9 @@ func (task *sBaremetalPrepareTask) prepareBaremetalInfo(cli *ssh.Client) (*barem
 	if len(raidDiskInfo) > 0 {
 		raidDrivers := []string{}
 		for _, drv := range raidDiskInfo {
-			raidDrivers = append(raidDrivers, drv.Driver)
+			if !utils.IsInStringArray(drv.Driver, raidDrivers) {
+				raidDrivers = append(raidDrivers, drv.Driver)
+			}
 		}
 		storageDriver = strings.Join(raidDrivers, ",")
 	} else {

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/app/auth.go

@@ -83,8 +83,9 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
-		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
-			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,
+		policy.EnableGlobalRbac(
+			time.Second*time.Duration(options.RbacPolicySyncPeriodSeconds),
+			time.Second*time.Duration(options.RbacPolicySyncFailedRetrySeconds),
 			options.RbacDebug,
 		)
 	}

pkg/cloudcommon/cmdline/helper.go

@@ -402,7 +402,7 @@ func FetchServerCreateInputByJSON(obj jsonutils.JSONObject) (*compute.ServerCrea
 		input.InstanceType = skuName
 	}
 	if keypair := jsonutils.GetAnyString(obj, []string{"keypair", "keypair_id"}); keypair != "" {
-		input.KeypairId = keypair
+		input.Keypair = keypair
 	}
 	if secgroup := jsonutils.GetAnyString(obj, []string{"secgroup", "secgroup_id", "secgrp_id"}); len(secgroup) != 0 {
 		input.SecgroupId = secgroup

pkg/cloudcommon/database.go

@@ -44,7 +44,7 @@ func InitDB(options *common_options.DBOptions) {
 
 	dialect, sqlStr, err := options.GetDBConnection()
 	if err != nil {
-		log.Fatalf("Invalid SqlConnection string: %s", options.SqlConnection)
+		log.Fatalf("Invalid SqlConnection string: %s error: %v", options.SqlConnection, err)
 	}
 	dbConn, err := sql.Open(dialect, sqlStr)
 	if err != nil {

pkg/cloudcommon/db/db_dispatcher.go

@@ -686,7 +686,7 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 			}
 		}
 		// do limit
-		if limit > 0 && total > limit {
+		if limit > 0 && total-offset > limit {
 			data = data[:limit]
 		}
 	}
@@ -1155,6 +1155,12 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 	model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId)
 	if err != nil {
 		// log.Errorf("fail to doCreateItem %s", err)
+		if CancelPendingUsagesInContext != nil {
+			err := CancelPendingUsagesInContext(ctx, userCred)
+			if err != nil {
+				log.Errorf("CancelPendingUsagesInContext fail %s", err)
+			}
+		}
 		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
 		if failErr != nil {
 			log.Errorf("manager.OnCreateFailed %s", failErr)
@@ -1347,6 +1353,19 @@ func managerPerformCheckCreateData(
 		return nil, httperrors.NewForbiddenError("not allow to perform %s", action)
 	}
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+
+		defer func() {
+			if CancelPendingUsagesInContext != nil {
+				err := CancelPendingUsagesInContext(ctx, userCred)
+				if err != nil {
+					log.Errorf("CancelPendingUsagesInContext fail %s", err)
+				}
+			}
+		}()
+	}
+
 	return ValidateCreateData(manager, ctx, userCred, ownerId, query, bodyDict)
 }
 

pkg/cloudcommon/db/modelbase.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -391,12 +390,6 @@ func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Cont
 }
 
 func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	if CancelPendingUsagesInContext != nil {
-		err := CancelPendingUsagesInContext(ctx, userCred)
-		if err != nil {
-			return errors.Wrap(err, "CancelPendingUsagesInContext")
-		}
-	}
 	return nil
 }
 

pkg/cloudcommon/db/proxy/proxysetting.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/net/http/httpproxy"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -87,8 +88,7 @@ func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context) error {
 		return httperrors.NewConflictError("DIRECT setting cannot be deleted")
 	}
 	for _, man := range referrersMen {
-		t := man.TableSpec().Instance()
-		n, err := t.Query().
+		n, err := man.Query().
 			Equals("proxy_setting_id", ps.Id).
 			CountWithError()
 		if err != nil {
@@ -106,19 +106,13 @@ func (ps *SProxySetting) AllowPerformTest(ctx context.Context, userCred mcclient
 	return db.IsAdminAllowPerform(userCred, ps, "test")
 }
 
-func (ps *SProxySetting) PerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (man *SProxySettingManager) test(ctx context.Context, urls map[string]string) (jsonutils.JSONObject, error) {
 	type TestURLResult struct {
 		Ok     bool   `json:"ok"`
 		Reason string `json:"reason"`
 	}
-	var (
-		r = map[string]TestURLResult{}
-		m = map[string]string{
-			"http_proxy":  ps.HTTPProxy,
-			"https_proxy": ps.HTTPSProxy,
-		}
-	)
-	for k, v := range m {
+	r := map[string]TestURLResult{}
+	for k, v := range urls {
 		if v == "" {
 			r[k] = TestURLResult{Ok: true}
 			continue
@@ -165,6 +159,22 @@ func (ps *SProxySetting) PerformTest(ctx context.Context, userCred mcclient.Toke
 	return jsonutils.Marshal(r), nil
 }
 
+func (ps *SProxySetting) PerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	urls := map[string]string{
+		"http_proxy":  ps.HTTPProxy,
+		"https_proxy": ps.HTTPSProxy,
+	}
+	return ProxySettingManager.test(ctx, urls)
+}
+
+func (man *SProxySettingManager) PerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data proxyapi.ProxySettingTestInput) (jsonutils.JSONObject, error) {
+	urls := map[string]string{
+		"http_proxy":  data.HttpProxy,
+		"https_proxy": data.HttpsProxy,
+	}
+	return man.test(ctx, urls)
+}
+
 func (man *SProxySettingManager) InitializeData() error {
 	_, err := man.FetchById(proxyapi.ProxySettingId_DIRECT)
 	if err == nil {
@@ -193,3 +203,16 @@ var referrersMen []db.IModelManager
 func RegisterReferrer(man db.IModelManager) {
 	referrersMen = append(referrersMen, man)
 }
+
+func ValidateProxySettingResourceInput(userCred mcclient.TokenCredential, input proxyapi.ProxySettingResourceInput) (*SProxySetting, proxyapi.ProxySettingResourceInput, error) {
+	m, err := ProxySettingManager.FetchByIdOrName(userCred, input.ProxySetting)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, input, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", ProxySettingManager.Keyword(), input.ProxySetting)
+		} else {
+			return nil, input, errors.Wrapf(err, "ProxySettingManager.FetchByIdOrName")
+		}
+	}
+	input.ProxySetting = m.GetId()
+	return m.(*SProxySetting), input, nil
+}

pkg/cloudcommon/db/quotas/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -435,7 +436,7 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) {
@@ -478,7 +479,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
@@ -566,3 +567,33 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 	}
 	return ret, nil
 }
+
+type tQuotaResultList []jsonutils.JSONObject
+
+func (a tQuotaResultList) Len() int           { return len(a) }
+func (a tQuotaResultList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a tQuotaResultList) Less(i, j int) bool { return usageRateOfQuota(a[i]) > usageRateOfQuota(a[j]) }
+
+func usageRateOfQuota(quota jsonutils.JSONObject) float32 {
+	maxRate := float32(0)
+	quotaMap, _ := quota.GetMap()
+	for k, v := range quotaMap {
+		usageK := fmt.Sprintf("usage.%s", k)
+		if usageV, ok := quotaMap[usageK]; ok {
+			intV, _ := v.Int()
+			if intV > 0 {
+				intUsageV, _ := usageV.Int()
+				rate := float32(intUsageV) / float32(intV)
+				if maxRate < rate {
+					maxRate = rate
+				}
+			}
+		}
+	}
+	return maxRate
+}
+
+func sortQuotaByUsage(quotaList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	sort.Sort(tQuotaResultList(quotaList))
+	return quotaList
+}

pkg/cloudcommon/db/rbac.go

@@ -70,7 +70,7 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	if !requireScope.HigherThan(scope) {
 		return nil
 	}
-	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privillege(require:%s,allow:%s)", requireScope, scope))
+	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privilege(require:%s,allow:%s)", requireScope, scope))
 }
 
 func isJointObjectRbacAllowed(item IJointModel, userCred mcclient.TokenCredential, action string, extra ...string) error {

pkg/cloudcommon/db/tenantcache.go

@@ -66,8 +66,8 @@ func RegistUserCredCacheUpdater() {
 	auth.RegisterAuthHook(onAuthCompleteUpdateCache)
 }
 
-func onAuthCompleteUpdateCache(userCred mcclient.TokenCredential) {
-	TenantCacheManager.updateTenantCache(userCred)
+func onAuthCompleteUpdateCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	TenantCacheManager.updateTenantCache(ctx, userCred)
 	UserCacheManager.updateUserCache(userCred)
 }
 
@@ -91,8 +91,8 @@ func (manager *STenantCacheManager) InitializeData() error {
 	return nil
 }
 
-func (manager *STenantCacheManager) updateTenantCache(userCred mcclient.TokenCredential) {
-	manager.Save(context.Background(), userCred.GetProjectId(), userCred.GetProjectName(),
+func (manager *STenantCacheManager) updateTenantCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	manager.Save(ctx, userCred.GetProjectId(), userCred.GetProjectName(),
 		userCred.GetProjectDomainId(), userCred.GetProjectDomain())
 }
 
@@ -348,14 +348,14 @@ func (manager *STenantCacheManager) findFirstProjectOfDomain(domainId string) (*
 	return &tenant, nil
 }
 
-func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId string) error {
+func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.Context, domainId string) error {
 	if len(domainId) == 0 {
 		log.Debugf("fetch empty domain!!!!")
 		debug.PrintStack()
 		return fmt.Errorf("Empty domainId")
 	}
 
-	s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
 	params := jsonutils.Marshal(map[string]string{"domain_id": domainId})
 	tenants, err := modules.Projects.List(s, params)
 	if err != nil {
@@ -366,7 +366,7 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 		tenantName, _ := tenant.GetString("name")
 		domainId, _ := tenant.GetString("domain_id")
 		domainName, _ := tenant.GetString("project_domain")
-		_, err = manager.Save(context.Background(), tenantId, tenantName, domainId, domainName)
+		_, err = manager.Save(ctx, tenantId, tenantName, domainId, domainName)
 		if err != nil {
 			return err
 		}
@@ -374,11 +374,11 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 	return nil
 }
 
-func (manager *STenantCacheManager) FindFirstProjectOfDomain(domainId string) (*STenant, error) {
+func (manager *STenantCacheManager) FindFirstProjectOfDomain(ctx context.Context, domainId string) (*STenant, error) {
 	tenant, err := manager.findFirstProjectOfDomain(domainId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			err = manager.fetchDomainTenantsFromKeystone(domainId)
+			err = manager.fetchDomainTenantsFromKeystone(ctx, domainId)
 			if err != nil {
 				return nil, errors.Wrap(err, "fetchDomainTenantsFromKeystone")
 			}

pkg/cloudcommon/policy/policy.go

@@ -20,6 +20,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -80,35 +81,42 @@ type SPolicyManager struct {
 	lock *sync.Mutex
 }
 
+type sPolicyData struct {
+	Type     string               `json:"type"`
+	Enabled  bool                 `json:"enabled"`
+	DomainId string               `json:"domain_id"`
+	IsPublic bool                 `json:"is_public"`
+	Policy   jsonutils.JSONObject `json:"policy"`
+}
+
 func parseJsonPolicy(obj jsonutils.JSONObject) (string, *rbacutils.SRbacPolicy, error) {
-	typeStr, err := obj.GetString("type")
+	pData := sPolicyData{}
+	err := obj.Unmarshal(&pData)
 	if err != nil {
-		return "", nil, errors.Wrap(err, "missing type")
+		return "", nil, errors.Wrap(err, "Unmarshal")
 	}
-	domainId, err := obj.GetString("domain_id")
-	if err != nil {
-		return "", nil, errors.Wrap(err, "missing domain_id")
+	if !pData.Enabled {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "not enabled")
+	}
+	if len(pData.Type) == 0 {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing type")
 	}
 
-	isPublic := jsonutils.QueryBoolean(obj, "is_public", false)
-
-	blob, err := obj.Get("policy")
-	if err != nil {
-		log.Errorf("get blob error %s", err)
-		return "", nil, errors.Wrap(err, "json.Get")
+	if pData.Policy == nil {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing policy")
 	}
 
 	policy := rbacutils.SRbacPolicy{}
-	err = policy.Decode(blob)
+	err = policy.Decode(pData.Policy)
 	if err != nil {
 		log.Errorf("policy decode error %s", err)
 		return "", nil, errors.Wrap(err, "policy.Decode")
 	}
 
-	policy.DomainId = domainId
-	policy.IsPublic = isPublic
+	policy.DomainId = pData.DomainId
+	policy.IsPublic = pData.IsPublic
 
-	return typeStr, &policy, nil
+	return pData.Type, &policy, nil
 }
 
 func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbacPolicy, error) {
@@ -121,7 +129,7 @@ func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbac
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewInt(2048), "limit")
 		params.Add(jsonutils.NewInt(int64(offset)), "offset")
-		params.Add(jsonutils.JSONTrue, "admin")
+		params.Add(jsonutils.NewString("system"), "scope")
 		params.Add(jsonutils.JSONTrue, "enabled")
 		result, err := modules.Policies.List(s, params)
 		if err != nil {
@@ -165,16 +173,38 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 			policiesMap[policy.Scope] = policies
 		}
 		manager.defaultPolicies = policiesMap
-		log.Debugf("%#v", manager.defaultPolicies)
+		// log.Debugf("%#v", manager.defaultPolicies)
 	}
 
 	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
 
-	manager.SyncOnce()
+	manager.syncByInterval()
 }
 
+func (manager *SPolicyManager) syncByInterval() {
+	syncWorkerManager.Run(manager.syncByInterval_, nil, nil)
+}
+
+func (manager *SPolicyManager) syncByInterval_() {
+	err := manager.doSync()
+	var interval time.Duration
+	if err != nil {
+		interval = manager.failedRetryInterval
+	} else {
+		interval = manager.refreshInterval
+	}
+	time.AfterFunc(interval, manager.syncByInterval)
+}
+
+var syncOnce int32
+
 func (manager *SPolicyManager) SyncOnce() {
-	syncWorkerManager.Run(manager.sync, nil, nil)
+	if atomic.CompareAndSwapInt32(&syncOnce, 0, 1) {
+		syncWorkerManager.Run(func() {
+			atomic.StoreInt32(&syncOnce, 0)
+			manager.doSync()
+		}, nil, nil)
+	}
 }
 
 func (manager *SPolicyManager) doSync() error {
@@ -202,17 +232,6 @@ func (manager *SPolicyManager) doSync() error {
 	return nil
 }
 
-func (manager *SPolicyManager) sync() {
-	err := manager.doSync()
-	var interval time.Duration
-	if err != nil {
-		interval = manager.failedRetryInterval
-	} else {
-		interval = manager.refreshInterval
-	}
-	time.AfterFunc(interval, manager.SyncOnce)
-}
-
 func queryKey(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) string {
 	queryKeys := []string{string(scope)}
 	queryKeys = append(queryKeys, userCred.GetProjectId(), userCred.GetDomainId(), userCred.GetUserId())
@@ -315,42 +334,16 @@ func (manager *SPolicyManager) findPolicyByName(scope rbacutils.TRbacScope, name
 }
 
 func getMatchedPolicyNames(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity) []string {
-	matchNames := make([]string, 0)
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchNames = matchNames[:0]
-		}
-		matchNames = append(matchNames, k)
-	}
+	_, matchNames := rbacutils.GetMatchedPolicies(policies, userCred)
 	return matchNames
 }
 
 func getMatchedPolicyRules(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity, service string, resource string, action string, extra ...string) ([]rbacutils.SRbacRule, bool) {
-	matchRules := make([]rbacutils.SRbacRule, 0)
-	findMatchPolicy := false
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchRules = matchRules[:0]
-		}
-		findMatchPolicy = true
-		rule := policies[k].GetMatchRule(service, resource, action, extra...)
-		if rule != nil {
-			matchRules = append(matchRules, *rule)
-		}
+	matchPolicies, _ := rbacutils.GetMatchedPolicies(policies, userCred)
+	if len(matchPolicies) == 0 {
+		return nil, false
 	}
-	return matchRules, findMatchPolicy
+	return matchPolicies.GetMatchRules(service, resource, action, extra...), true
 }
 
 func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
@@ -406,8 +399,15 @@ func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, use
 
 	var result rbacutils.TRbacResult
 	if len(matchRules) > 0 {
-		rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
-		result = rule.Result
+		result = rbacutils.Deny
+		for _, rule := range matchRules {
+			if rule.Result == rbacutils.Allow {
+				result = rbacutils.Allow
+				break
+			}
+		}
+		// rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
+		// result = rule.Result
 	} else if findMatchPolicy {
 		// if find matched policy, but no rule matching, allow anyway
 		result = rbacutils.Allow
@@ -520,7 +520,7 @@ func (manager *SPolicyManager) IsScopeCapable(userCred mcclient.TokenCredential,
 	return false
 }
 
-func (manager *SPolicyManager) MatchedPolicies(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
+func (manager *SPolicyManager) MatchedPolicyNames(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
 	ret := make([]string, 0)
 	policies, ok := manager.policies[scope]
 	if !ok {
@@ -544,13 +544,28 @@ func (manager *SPolicyManager) AllPolicies() map[string][]string {
 }
 
 func (manager *SPolicyManager) RoleMatchPolicies(roleName string) []string {
+	ident := rbacutils.NewRbacIdentity("", "", []string{roleName})
 	ret := make([]string, 0)
 	for _, policies := range manager.policies {
 		for name, policy := range policies {
-			if policy.MatchRole(roleName) {
+			if matched, _ := policy.Match(ident); matched {
 				ret = append(ret, name)
 			}
 		}
 	}
 	return ret
 }
+
+func (manager *SPolicyManager) GetMatchedPolicySet(userCred rbacutils.IRbacIdentity) (rbacutils.TRbacScope, rbacutils.TPolicySet) {
+	for _, scope := range []rbacutils.TRbacScope{
+		rbacutils.ScopeSystem,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeProject,
+	} {
+		macthed, _ := rbacutils.GetMatchedPolicies(manager.policies[scope], userCred)
+		if len(macthed) > 0 {
+			return scope, macthed
+		}
+	}
+	return rbacutils.ScopeNone, nil
+}

pkg/cloudcommon/policy/resources.go

@@ -66,7 +66,9 @@ var (
 	meterDomainResources = []string{}
 	meterUserResources   = []string{}
 
-	k8sSystemResources = []string{}
+	k8sSystemResources = []string{
+		"repos",
+	}
 	k8sDomainResources = []string{}
 	k8sUserResources   = []string{}
 

pkg/cloudevent/models/cloudevents.go

@@ -86,19 +86,6 @@ func (manager *SCloudeventManager) ListItemFilter(ctx context.Context, q *sqlche
 		return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
 	}
 
-	/* input.Cloudprovider = input.CloudproviderStr()
-	if len(input.Cloudprovider) > 0 {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(userCred, input.Cloudprovider)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), input.Cloudprovider)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		q = q.Equals("cloudprovider_id", providerObj.GetId())
-	}*/
-
 	if len(input.Providers) > 0 {
 		q = q.In("provider", input.Providers)
 	}
@@ -107,6 +94,22 @@ func (manager *SCloudeventManager) ListItemFilter(ctx context.Context, q *sqlche
 		q = q.In("brand", input.Brands)
 	}
 
+	if len(input.Service) > 0 {
+		q = q.In("service", input.Service)
+	}
+
+	if len(input.Manager) > 0 {
+		q = q.In("manager", input.Manager)
+	}
+
+	if len(input.Account) > 0 {
+		q = q.In("account", input.Account)
+	}
+
+	if len(input.Action) > 0 {
+		q = q.In("action", input.Action)
+	}
+
 	if !input.Since.IsZero() {
 		q = q.GT("created_at", input.Since)
 	}

pkg/cloudevent/models/cloudproviders.go

@@ -36,8 +36,10 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudevent/options"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/s3gateway/session"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 type SCloudproviderManager struct {
@@ -61,39 +63,27 @@ func init() {
 type SCloudprovider struct {
 	db.SEnabledStatusStandaloneResourceBase
 
-	HealthStatus   string `width:"16" charset:"ascii" default:"normal" nullable:"false" list:"domain"`
 	SyncStatus     string
 	LastSync       time.Time
 	LastSyncEndAt  time.Time
 	LastSyncTimeAt time.Time
 
-	AccessUrl string `width:"64" charset:"ascii" nullable:"true" list:"domain" update:"domain"`
-	Account   string `width:"128" charset:"ascii" nullable:"false" list:"domain"`
-	Secret    string `length:"0" charset:"ascii" nullable:"false" list:"domain"`
-
 	Provider string `width:"64" charset:"ascii" list:"domain"`
 	Brand    string `width:"64" charset:"ascii" list:"domain"`
-
-	ProxySetting *proxyapi.SProxySetting
 }
 
 func (manager *SCloudproviderManager) GetRegionCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) ([]SCloudprovider, error) {
 	s := session.GetSession(ctx, userCred)
 	params := jsonutils.NewDict()
-	params.Add(jsonutils.JSONTrue, "details")
 	result, err := modules.Cloudproviders.List(s, params)
 	if err != nil {
 		return nil, errors.Wrap(err, "modules.Cloudproviders.List")
 	}
+
 	providers := []SCloudprovider{}
-	for _, _provider := range result.Data {
-		provider := SCloudprovider{}
-		provider.SetModelManager(manager, &provider)
-		err = _provider.Unmarshal(&provider)
-		if err != nil {
-			return nil, errors.Wrap(err, "_provider.Unmarshal")
-		}
-		providers = append(providers, provider)
+	err = jsonutils.Update(&providers, result.Data)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.Update")
 	}
 	return providers, nil
 }
@@ -108,17 +98,18 @@ func (manager *SCloudproviderManager) GetLocalCloudproviders() ([]SCloudprovider
 	return dbProviders, nil
 }
 
-func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+func (manager *SCloudproviderManager) syncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
+	result := compare.SyncResult{}
 	providers, err := manager.GetRegionCloudproviders(ctx, userCred)
 	if err != nil {
-		log.Errorf("failed to get region cloudproviders: %v", err)
-		return
+		result.Error(errors.Wrap(err, "GetRegionCloudproviders"))
+		return result
 	}
 
 	dbProviders, err := manager.GetLocalCloudproviders()
 	if err != nil {
-		log.Errorf("failed to get local cloudproviders: %v", err)
-		return
+		result.Error(errors.Wrap(err, "GetLocalCloudproviders"))
+		return result
 	}
 
 	removed := make([]SCloudprovider, 0)
@@ -128,36 +119,48 @@ func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, us
 
 	err = compare.CompareSets(dbProviders, providers, &removed, &commondb, &commonext, &added)
 	if err != nil {
-		log.Errorf("compare.CompareSets: %v", err)
-		return
+		result.Error(errors.Wrap(err, "CompareSets"))
+		return result
 	}
 
 	for i := 0; i < len(removed); i++ {
 		err = removed[i].Delete(ctx, userCred)
 		if err != nil {
-			log.Errorf("failed to remove cloudprovider %s(%s) error: %v", removed[i].Name, removed[i].Id, err)
+			result.DeleteError(err)
+			continue
 		}
+		result.Delete()
 	}
 
 	for i := 0; i < len(commondb); i++ {
 		err = commondb[i].syncWithRegionProvider(ctx, userCred, commonext[i])
 		if err != nil {
-			log.Errorf("failed to sync cloudprovider %s(%s) error: %v", commondb[i].Name, commondb[i].Id, err)
+			result.UpdateError(err)
+			continue
 		}
+		result.Update()
 	}
 
 	for i := 0; i < len(added); i++ {
 		err = manager.newFromRegionProvider(ctx, userCred, added[i])
 		if err != nil {
-			log.Errorf("failed to add cloudprovider %s(%s) error: %v", added[i].Name, added[i].Id, err)
+			result.AddError(err)
+			continue
 		}
+		result.Add()
 	}
+	return result
+}
+
+func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	result := manager.syncCloudproviders(ctx, userCred)
+	info := result.Result()
+	log.Infof("sync cloudproviders result: %s", info)
 }
 
 func (provider *SCloudprovider) syncWithRegionProvider(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider SCloudprovider) error {
 	_, err := db.Update(provider, func() error {
 		provider.Status = cloudprovider.Status
-		provider.Secret = cloudprovider.Secret
 		provider.Enabled = cloudprovider.Enabled
 		provider.Brand = cloudprovider.Brand
 		return nil
@@ -173,8 +176,7 @@ func (self *SCloudprovider) MarkSyncing(userCred mcclient.TokenCredential) error
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to MarkSyncing error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -186,8 +188,7 @@ func (self *SCloudprovider) MarkEndSync(userCred mcclient.TokenCredential) error
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to markEndSync error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -198,8 +199,7 @@ func (self *SCloudprovider) SetLastSyncTimeAt(userCred mcclient.TokenCredential,
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to SetLastSyncTimeAt error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -209,30 +209,36 @@ func (manager *SCloudproviderManager) newFromRegionProvider(ctx context.Context,
 	return manager.TableSpec().Insert(&cloudprovider)
 }
 
-func (manager *SCloudproviderManager) SyncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+func (manager *SCloudproviderManager) syncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential) error {
 	cloudproviders := []SCloudprovider{}
 	q := manager.Query().IsTrue("enabled").Equals("status", api.CLOUD_PROVIDER_CONNECTED).Equals("sync_status", api.CLOUD_PROVIDER_SYNC_STATUS_IDLE)
 	err := db.FetchModelObjects(manager, q, &cloudproviders)
 	if err != nil {
-		log.Errorf("failed to fetch cloudproviders")
-		return
+		return errors.Wrap(err, "db.FetchModelObjects")
 	}
-	for _, provider := range cloudproviders {
-		err = provider.StartCloudeventSyncTask(ctx, userCred)
+	for i := range cloudproviders {
+		err = cloudproviders[i].StartCloudeventSyncTask(ctx, userCred)
 		if err != nil {
-			log.Errorf("Failed start cloudevent sync task error: %v", err)
+			log.Errorf("Failed start cloudevent sync task for cloudprovider %s (%s) error: %v", cloudproviders[i].Name, cloudproviders[i].Id, err)
 		}
 	}
-	return
+	return nil
+}
+
+func (manager *SCloudproviderManager) SyncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	err := manager.syncCloudeventTask(ctx, userCred)
+	if err != nil {
+		log.Errorf("syncCloudeventTask error: %v", err)
+	}
 }
 
 func (provider *SCloudprovider) StartCloudeventSyncTask(ctx context.Context, userCred mcclient.TokenCredential) error {
 	params := jsonutils.NewDict()
-	provider.MarkSyncing(userCred)
 	task, err := taskman.TaskManager.NewTask(ctx, "CloudeventSyncTask", provider, userCred, params, "", "", nil)
 	if err != nil {
 		return errors.Wrap(err, "NewTask")
 	}
+	provider.MarkSyncing(userCred)
 	task.ScheduleRun(nil)
 	return nil
 }
@@ -277,11 +283,43 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	return start, end, nil
 }
 
-func (provider *SCloudprovider) getPassword() (string, error) {
+type SCloudproviderDelegate struct {
+	Id   string
+	Name string
+
+	Enabled    bool
+	Status     string
+	SyncStatus string
+
+	AccessUrl string
+	Account   string
+	Secret    string
+
+	Provider string
+	Brand    string
+
+	ProxySetting proxyapi.SProxySetting
+}
+
+func (self *SCloudprovider) GetDelegate() (*SCloudproviderDelegate, error) {
+	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	result, err := modules.Cloudproviders.Get(s, self.Id, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "modules.Cloudproviders.Get")
+	}
+	provider := &SCloudproviderDelegate{}
+	err = result.Unmarshal(provider)
+	if err != nil {
+		return nil, errors.Wrap(err, "result.Unmarshal")
+	}
+	return provider, nil
+}
+
+func (provider *SCloudproviderDelegate) getPassword() (string, error) {
 	return utils.DescryptAESBase64(provider.Id, provider.Secret)
 }
 
-func (provider *SCloudprovider) getAccessUrl() string {
+func (provider *SCloudproviderDelegate) getAccessUrl() string {
 	return provider.AccessUrl
 }
 
@@ -297,33 +335,40 @@ func (provider SCloudprovider) GetExternalId() string {
 	return provider.Id
 }
 
-func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) {
-	if !provider.Enabled {
+func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) {
+	if !self.Enabled {
 		return nil, errors.Error("Cloud provider is not enabled")
 	}
-	accessUrl := provider.getAccessUrl()
-	passwd, err := provider.getPassword()
+	delegate, err := self.GetDelegate()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "GetDelegate")
+	}
+	accessUrl := delegate.getAccessUrl()
+	passwd, err := delegate.getPassword()
+	if err != nil {
+		return nil, errors.Wrap(err, "delegate.getPassword")
 	}
 
-	ps := provider.ProxySetting
-	cfg := &httpproxy.Config{
-		HTTPProxy:  ps.HTTPProxy,
-		HTTPSProxy: ps.HTTPSProxy,
-		NoProxy:    ps.NoProxy,
-	}
-	cfgProxyFunc := cfg.ProxyFunc()
-	proxyFunc := func(req *http.Request) (*url.URL, error) {
-		return cfgProxyFunc(req.URL)
+	var proxyFunc httputils.TransportProxyFunc
+	{
+		cfg := &httpproxy.Config{
+			HTTPProxy:  delegate.ProxySetting.HTTPProxy,
+			HTTPSProxy: delegate.ProxySetting.HTTPSProxy,
+			NoProxy:    delegate.ProxySetting.NoProxy,
+		}
+		cfgProxyFunc := cfg.ProxyFunc()
+		proxyFunc = func(req *http.Request) (*url.URL, error) {
+			return cfgProxyFunc(req.URL)
+		}
 	}
+
 	return cloudprovider.GetProvider(
 		cloudprovider.ProviderConfig{
-			Id:      provider.Id,
-			Name:    provider.Name,
-			Vendor:  provider.Provider,
+			Id:      self.Id,
+			Name:    self.Name,
+			Vendor:  self.Provider,
 			URL:     accessUrl,
-			Account: provider.Account,
+			Account: delegate.Account,
 			Secret:  passwd,
 
 			ProxyFunc: proxyFunc,

pkg/cloudprovider/cloudprovider.go

@@ -17,6 +17,8 @@ package cloudprovider
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -33,66 +35,66 @@ const (
 )
 
 type SCloudaccountCredential struct {
-	// 账号所在的项目
+	// 账号所在的项目 (openstack)
 	ProjectName string `json:"project_name"`
 
-	// 账号所在的域
+	// 账号所在的域 (openstack)
 	// default: Default
 	DomainName string `json:"domain_name"`
 
-	// 用户名
+	// 用户名 (openstack, zstack, esxi)
 	Username string `json:"username"`
 
-	// 密码
+	// 密码 (openstack, zstack, esxi)
 	Password string `json:"password"`
 
-	// 认证地址
+	// 认证地址 (openstack,zstack)
 	AuthUrl string `json:"auto_url"`
 
-	// 秘钥id
+	// 秘钥id (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
 	AccessKeyId string `json:"access_key_id"`
 
-	// 秘钥key
+	// 秘钥key (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
 	AccessKeySecret string `json:"access_key_secret"`
 
-	// 环境
+	// 环境 (Azure, Aws, huawei, ctyun)
 	Environment string `json:"environment"`
 
-	// 目录ID
+	// 目录ID (Azure)
 	DirectoryId string `json:"directory_id"`
 
-	// 客户端ID
+	// 客户端ID (Azure)
 	ClientId string `json:"client_id"`
 
-	// 客户端秘钥
+	// 客户端秘钥 (Azure)
 	ClientSecret string `json:"client_secret"`
 
-	// 主机IP
+	// 主机IP (esxi)
 	Host string `json:"host"`
 
-	// 主机端口
+	// 主机端口 (esxi)
 	Port int `json:"port"`
 
-	// 端点
+	// 端点 (s3)
 	Endpoint string `json:"endpoint"`
 
-	// app id
+	// app id (Qcloud)
 	AppId string `json:"app_id"`
 
-	//秘钥ID
+	//秘钥ID (Qcloud)
 	SecretId string `json:"secret_id"`
 
-	//秘钥key
+	//秘钥key (Qcloud)
 	SecretKey string `json:"secret_key"`
 
-	// Google服务账号email
-	ClientEmail string `json:"client_email"`
-	// Google服务账号project id
-	ProjectId string `json:"project_id"`
-	// Google服务账号秘钥id
-	PrivateKeyId string `json:"private_key_id"`
-	// Google服务账号秘钥
-	PrivateKey string `json:"private_key"`
+	// Google服务账号email (gcp)
+	GCPClientEmail string `json:"gcp_client_email"`
+	// Google服务账号project id (gcp)
+	GCPProjectId string `json:"gcp_project_id"`
+	// Google服务账号秘钥id (gcp)
+	GCPPrivateKeyId string `json:"gcp_private_key_id"`
+	// Google服务账号秘钥 (gcp)
+	GCPPrivateKey string `json:"gcp_private_key"`
 }
 
 type SCloudaccount struct {
@@ -155,6 +157,18 @@ type ProviderConfig struct {
 	ProxyFunc httputils.TransportProxyFunc
 }
 
+func (cp *ProviderConfig) HttpClient() *http.Client {
+	client := httputils.GetClient(true, 15*time.Second)
+	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
+	return client
+}
+
+func (cp *ProviderConfig) AdaptiveTimeoutHttpClient() *http.Client {
+	client := httputils.GetAdaptiveTimeoutClient()
+	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
+	return client
+}
+
 type ICloudProviderFactory interface {
 	GetProvider(cfg ProviderConfig) (ICloudProvider, error)
 
@@ -243,7 +257,7 @@ func GetProviderFactory(provider string) (ICloudProviderFactory, error) {
 	if ok {
 		return factory, nil
 	}
-	log.Errorf("Provider %s not registerd", provider)
+	log.Errorf("Provider %s not registered", provider)
 	return nil, fmt.Errorf("No such provider %s", provider)
 }
 

pkg/cloudprovider/instance.go

@@ -37,6 +37,8 @@ const (
 	CLOUD_SHELL                 = "cloud-shell"
 	CLOUD_SHELL_WITHOUT_ENCRYPT = "cloud-shell-without-encrypt"
 	CLOUD_CONFIG                = "cloud-config"
+	CLOUD_POWER_SHELL           = "powershell"
+	CLOUD_EC2                   = "ec2"
 )
 
 type SManagedVMCreateConfig struct {

pkg/cloudprovider/objectstore.go

@@ -270,6 +270,7 @@ func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) {
 	if objs.IsTruncated {
 		return stats, errors.Wrap(httperrors.ErrTooLarge, "too many objects")
 	}
+	stats.ObjectCount, stats.SizeBytes = 0, 0
 	for _, obj := range objs.Objects {
 		stats.SizeBytes += obj.GetSizeBytes()
 		stats.ObjectCount += 1

pkg/cloudprovider/resources.go

@@ -20,7 +20,6 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/tristate"
-	"yunion.io/x/pkg/util/secrules"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
@@ -338,10 +337,10 @@ type ICloudSecurityGroup interface {
 	ICloudResource
 
 	GetDescription() string
-	GetRules() ([]secrules.SecurityRule, error)
+	GetRules() ([]SecurityRule, error)
 	GetVpcId() string
 
-	SyncRules(rules []secrules.SecurityRule) error
+	SyncRules(common, inAdds, outAdds, inDels, outDels []SecurityRule) error
 	Delete() error
 }
 
@@ -750,7 +749,9 @@ type ICloudDBInstance interface {
 
 	GetConnectionStr() string
 	GetInternalConnectionStr() string
-	GetIZoneId() string
+	GetZone1Id() string
+	GetZone2Id() string
+	GetZone3Id() string
 	GetIVpcId() string
 
 	GetDBNetwork() (*SDBInstanceNetwork, error)

pkg/cloudprovider/securitygroup.go

@@ -14,7 +14,15 @@
 
 package cloudprovider
 
-import "yunion.io/x/pkg/util/secrules"
+import (
+	"sort"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/secrules"
+)
+
+const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
 
 type SecurityGroupCreateInput struct {
 	Name  string
@@ -22,3 +30,268 @@ type SecurityGroupCreateInput struct {
 	VpcId string
 	Rules []secrules.SecurityRule
 }
+
+type SecurityRule struct {
+	secrules.SecurityRule
+	Name       string
+	ExternalId string
+}
+
+type TPriorityOrder int
+
+var (
+	PriorityOrderByDesc = TPriorityOrder(1)
+	PriorityOrderByAsc  = TPriorityOrder(-1)
+)
+
+func (r SecurityRule) String() string {
+	return r.SecurityRule.String()
+}
+
+type SecurityRuleSet []SecurityRule
+
+func (srs SecurityRuleSet) Len() int {
+	return len(srs)
+}
+
+func (srs SecurityRuleSet) Swap(i, j int) {
+	srs[i], srs[j] = srs[j], srs[i]
+}
+
+func (srs SecurityRuleSet) Less(i, j int) bool {
+	return srs[i].Priority < srs[j].Priority || (srs[i].Priority == srs[j].Priority && srs[i].String() < srs[j].String())
+}
+
+func (srs SecurityRuleSet) AllowList() secrules.SecurityRuleSet {
+	rules := secrules.SecurityRuleSet{}
+	for _, r := range srs {
+		rules = append(rules, r.SecurityRule)
+	}
+	return rules.AllowList()
+}
+
+func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr string, order TPriorityOrder, min, max int, onlyAllowRules bool) []SecurityRule {
+	if defaultRule.String() == localRuleStr {
+		return rules
+	}
+	defaultRule.ExternalId = DEFAULT_CLOUD_RULE_ID
+	if order == PriorityOrderByDesc {
+		defaultRule.Priority = min
+	} else {
+		defaultRule.Priority = max
+	}
+	defaultRule.Priority -= int(order)
+	if onlyAllowRules {
+		defaultRule.Priority = -1
+	}
+	return append(rules, defaultRule)
+}
+
+func SortSecurityRule(rules SecurityRuleSet, order TPriorityOrder, onlyAllowRules bool) {
+	if onlyAllowRules {
+		sort.Sort(rules)
+		return
+	}
+	if order == PriorityOrderByAsc {
+		sort.Sort(sort.Reverse(rules))
+		return
+	}
+	sort.Sort(rules)
+}
+
+func CompareRules(
+	minPriority, maxPriority int, order TPriorityOrder,
+	localRules secrules.SecurityRuleSet, remoteRules []SecurityRule,
+	defaultInRule, defaultOutRule SecurityRule,
+	onlyAllowRules bool, debug bool,
+) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
+	localInRules := secrules.SecurityRuleSet{}
+	localOutRules := secrules.SecurityRuleSet{}
+	for i := range localRules {
+		if localRules[i].Direction == secrules.DIR_IN {
+			localInRules = append(localInRules, localRules[i])
+		} else {
+			localOutRules = append(localOutRules, localRules[i])
+		}
+	}
+	inRules := SecurityRuleSet{}
+	outRules := SecurityRuleSet{}
+	for i := 0; i < len(remoteRules); i++ {
+		if remoteRules[i].Direction == secrules.DIR_IN {
+			inRules = append(inRules, remoteRules[i])
+		} else {
+			outRules = append(outRules, remoteRules[i])
+		}
+	}
+	var inCommon, outCommon = inRules, outRules
+
+	defaultLocalInRule := *secrules.MustParseSecurityRule("in:deny any")
+	defaultLocalOutRule := *secrules.MustParseSecurityRule("out:allow any")
+
+	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+
+	if defaultLocalInRule.String() != defaultInRule.String() {
+		localInRules = append(localInRules, defaultLocalInRule)
+	}
+	if defaultLocalOutRule.String() != defaultOutRule.String() {
+		localOutRules = append(localOutRules, defaultLocalOutRule)
+	}
+
+	sort.Sort(localInRules)
+	sort.Sort(localOutRules)
+
+	localInAllowList := localInRules.AllowList()
+	localOutAllowList := localOutRules.AllowList()
+	if onlyAllowRules {
+		localInRules = localInAllowList
+		localOutRules = localOutAllowList
+	}
+
+	SortSecurityRule(inRules, order, onlyAllowRules)
+	SortSecurityRule(outRules, order, onlyAllowRules)
+
+	inAllowList := inRules.AllowList()
+	outAllowList := outRules.AllowList()
+	inEquals, outEquals := inAllowList.Equals(localInAllowList), outAllowList.Equals(localOutAllowList)
+	if inEquals && outEquals {
+		return
+	}
+
+	// priority从小到大排列(从默认规则开始对比)
+	sort.Sort(sort.Reverse(localInRules))
+	sort.Sort(sort.Reverse(localOutRules))
+
+	sort.Sort(sort.Reverse(inRules))
+	sort.Sort(sort.Reverse(outRules))
+
+	startPriority := minPriority - 1
+	if order == PriorityOrderByAsc {
+		startPriority = maxPriority + 1
+	}
+
+	var addPriority = func(priority int, order TPriorityOrder, inc int, min, max int, onlyAllowRules bool) int {
+		if onlyAllowRules {
+			return 0
+		}
+		inc = inc * int(order) //+ int(order)
+		priority += inc
+		if priority < min {
+			return min
+		}
+		if priority > max {
+			return max
+		}
+		return priority
+	}
+
+	var getInitPriority = func(init, min, max int) int {
+		if init < min || init > max {
+			return (min + max) / 2
+		}
+		return init
+	}
+
+	var compare = func(localRules secrules.SecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
+		i, j, inc, prePriority := 0, 0, 1, 0
+		for i < len(localRules) || j < len(remoteRules) {
+			if i < len(localRules) && j < len(remoteRules) {
+				ruleStr := remoteRules[j].String()
+				localRuleStr := localRules[i].String()
+				if debug {
+					log.Debugf("compare local priority(%d) %s -> remote name(%s) priority(%d) %s\n", localRules[i].Priority, localRules[i].String(), remoteRules[j].Name, remoteRules[j].Priority, remoteRules[j].String())
+				}
+				cmp := strings.Compare(ruleStr, localRuleStr)
+				if cmp == 0 {
+					prePriority = remoteRules[j].Priority
+					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
+						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
+					}
+					common = append(common, remoteRules[j])
+					i++
+					j++
+				} else if cmp < 0 {
+					if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+						del = append(del, remoteRules[j])
+					}
+					j++
+				} else {
+					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
+					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+					add = append(add, SecurityRule{SecurityRule: localRules[i]})
+					i++
+					inc++
+				}
+			} else if i >= len(localRules) {
+				if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+					del = append(del, remoteRules[j])
+				}
+				j++
+			} else if j >= len(remoteRules) {
+				initPriority := startPriority
+				if len(remoteRules) > 0 {
+					initPriority = remoteRules[len(remoteRules)-1].Priority
+				}
+				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
+				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+				add = append(add, SecurityRule{SecurityRule: localRules[i]})
+				i++
+				inc++
+			}
+		}
+		return
+	}
+
+	type rulePair struct {
+		localRules  []secrules.SecurityRule
+		remoteRules []SecurityRule
+		protocol    string
+	}
+
+	var splitRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) []rulePair {
+		rules := map[string]rulePair{}
+		for _, r := range localRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.localRules = append(pair.localRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		for _, r := range remoteRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.remoteRules = append(pair.remoteRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		ret := []rulePair{}
+		for _, r := range rules {
+			ret = append(ret, r)
+		}
+		return ret
+	}
+
+	var compareRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
+		pairs := splitRules(localRules, remoteRules)
+		for _, r := range pairs {
+			_common, _add, _dels := compare(r.localRules, r.remoteRules)
+			common = append(common, _common...)
+			add = append(add, _add...)
+			dels = append(dels, _dels...)
+		}
+		return
+	}
+
+	if !inEquals {
+		inCommon, inAdds, inDels = compareRules(localInRules, inRules)
+	}
+	if !outEquals {
+		outCommon, outAdds, outDels = compareRules(localOutRules, outRules)
+	}
+	common = append(inCommon, outCommon...)
+	return
+}

pkg/compute/baremetal/diskconfig.go

@@ -549,7 +549,7 @@ func GetDiskSpecV2(storages []*BaremetalStorage) api.DiskDriverSpec {
 		if len(driverStorages) == 0 {
 			continue
 		}
-		spec[driver] = getSpec(storages)
+		spec[driver] = getSpec(driverStorages)
 	}
 	return spec
 }

pkg/compute/guestdrivers/aws.go

@@ -78,6 +78,14 @@ func (self *SAwsGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovider
 	return true
 }
 
+func (self *SAwsGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
+func (self *SAwsGuestDriver) GetWindowsUserDataType() string {
+	return cloudprovider.CLOUD_EC2
+}
+
 func (self *SAwsGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	// return fetchAwsUserName(desc)
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/base.go

@@ -298,6 +298,14 @@ func (self *SBaseGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovide
 	return false
 }
 
+func (self *SBaseGuestDriver) GetWindowsUserDataType() string {
+	return cloudprovider.CLOUD_POWER_SHELL
+}
+
+func (self *SBaseGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return false
+}
+
 func (self *SBaseGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_CONFIG
 }

pkg/compute/guestdrivers/esxi.go

@@ -172,7 +172,7 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 	if img.ImageType != cloudprovider.CachedImageTypeSystem {
 		return desc
 	}
-	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).SubQuery()
+	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_READY).SubQuery()
 	scQ := models.StoragecacheManager.Query().In("id", sciSubQ)
 	storageCaches := make([]models.SStoragecache, 0, 1)
 	err = db.FetchModelObjects(models.StoragecacheManager, scQ, &storageCaches)
@@ -346,3 +346,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/guestdrivers/managedvirtual.go

@@ -16,6 +16,7 @@ package guestdrivers
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"math"
 	"strings"
@@ -346,7 +347,15 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context
 			desc.UserData = oUserData.UserDataBase64()
 		}
 		if strings.ToLower(desc.OsType) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) {
-			desc.UserData = oUserData.UserDataPowerShell()
+			switch guest.GetDriver().GetWindowsUserDataType() {
+			case cloudprovider.CLOUD_EC2:
+				desc.UserData = oUserData.UserDataEc2()
+			default:
+				desc.UserData = oUserData.UserDataPowerShell()
+			}
+			if guest.GetDriver().IsWindowsUserDataTypeNeedEncode() {
+				desc.UserData = base64.StdEncoding.EncodeToString([]byte(desc.UserData))
+			}
 		}
 	}
 
@@ -462,20 +471,20 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte
 		return nil, err
 	}
 
+	ret, expect := 0, len(desc.DataDisks)+1
 	err = cloudprovider.RetryUntil(func() (bool, error) {
 		idisks, err := iVM.GetIDisks()
 		if err != nil {
-			log.Errorf("fail to find vm disks %s after create", err)
-			return false, err
+			return false, errors.Wrap(err, "iVM.GetIDisks")
 		}
-		if len(idisks) == len(desc.DataDisks)+1 {
+		ret = len(idisks)
+		if ret >= expect { // 有可能自定义镜像里面也有磁盘，会导致返回的磁盘多于创建时的磁盘
 			return true, nil
-		} else {
-			return false, nil
 		}
+		return false, nil
 	}, 10)
 	if err != nil {
-		return nil, errors.Wrap(err, "GuestDriver.RemoteDeployGuestForCreate.RetryUntil")
+		return nil, errors.Wrapf(err, "GuestDriver.RemoteDeployGuestForCreate.RetryUntil expect %d disks return %d disks", expect, ret)
 	}
 
 	guest.GetDriver().RemoteActionAfterGuestCreated(ctx, userCred, guest, host, iVM, &desc)
@@ -1033,7 +1042,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestAssociateEip(ctx context.Cont
 			return nil, fmt.Errorf("ManagedVirtualizedGuestDriver.RequestAssociateEip fail to local associate EIP %s", err)
 		}
 
-		eip.SetStatus(userCred, api.EIP_STATUS_READY, "associate")
+		eip.SetStatus(userCred, api.EIP_STATUS_READY, api.EIP_STATUS_ASSOCIATE)
 		return nil, nil
 	})
 

pkg/compute/guestdrivers/zstack.go

@@ -154,6 +154,10 @@ func (self *SZStackGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }
 
+func (self *SZStackGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
 func (self *SZStackGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/models/buckets.go

@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	identity_apis "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -535,7 +536,7 @@ func joinPath(ep, path string) string {
 func (bucket *SBucket) getMoreDetails(out api.BucketDetails) api.BucketDetails {
 	out.CloudproviderInfo = bucket.getCloudProviderInfo()
 
-	s3gwUrl, _ := auth.GetServiceURL("s3gateway", options.Options.Region, "", "public")
+	s3gwUrl, _ := auth.GetServiceURL("s3gateway", options.Options.Region, "", identity_apis.EndpointInterfacePublic)
 	if len(s3gwUrl) > 0 {
 		accessUrls := make([]cloudprovider.SBucketAccessUrl, 0)
 		if bucket.AccessUrls != nil {
@@ -654,10 +655,10 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	ret := jsonutils.NewDict()
 	ret.Add(retArray, "data")
+	ret.Add(jsonutils.NewString("key"), "marker_field")
+	ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	if len(nextMarker) > 0 {
 		ret.Add(jsonutils.NewString(nextMarker), "next_marker")
-		ret.Add(jsonutils.NewString("key"), "marker_field")
-		ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	}
 	return ret, nil
 }

pkg/compute/models/cloudaccounts.go

@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -318,10 +319,26 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl
 		"proxy_setting",
 		proxy.ProxySettingManager.Keyword(),
 		userCred,
-	).Optional(true)
+	)
+	v.Optional(true)
 	if err := v.Validate(data); err != nil {
 		return nil, err
 	}
+	if proxySetting, ok := v.Model.(*proxy.SProxySetting); ok && proxySetting.Id != self.ProxySettingId {
+		// updated proxy setting, so do the check
+		proxyFunc := proxySetting.HttpTransportProxyFunc()
+		secret, _ := self.getPassword()
+		_, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
+			Vendor:    self.Provider,
+			URL:       self.AccessUrl,
+			Account:   self.Account,
+			Secret:    secret,
+			ProxyFunc: proxyFunc,
+		})
+		if err != nil {
+			return nil, httperrors.NewInputParameterError("invalid proxy setting %s", err)
+		}
+	}
 
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }
@@ -382,16 +399,14 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 
 	var proxyFunc httputils.TransportProxyFunc
 	{
-		if input.ProxySettingId == "" {
-			input.ProxySettingId = proxyapi.ProxySettingId_DIRECT
+		if input.ProxySetting == "" {
+			input.ProxySetting = proxyapi.ProxySettingId_DIRECT
 		}
-		m, err := proxy.ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId)
+		var proxySetting *proxy.SProxySetting
+		proxySetting, input.ProxySettingResourceInput, err = proxy.ValidateProxySettingResourceInput(userCred, input.ProxySettingResourceInput)
 		if err != nil {
-			return input, httperrors.NewInputParameterError("fetch proxysetting %s: %s",
-				input.ProxySettingId, err)
+			return input, httperrors.NewInputParameterError("ValidateProxySettingResourceInput %s", err)
 		}
-		proxySetting := m.(*proxy.SProxySetting)
-		input.ProxySettingId = proxySetting.Id
 		proxyFunc = proxySetting.HttpTransportProxyFunc()
 	}
 	accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
@@ -624,13 +639,13 @@ func (self *SCloudaccount) StartSyncCloudProviderInfoTask(ctx context.Context, u
 		log.Errorf("CloudAccountSyncInfoTask newTask error %s", err)
 		return err
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
 }
 
-func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -642,7 +657,7 @@ func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) erro
 	providers := self.GetCloudproviders()
 	for i := range providers {
 		if providers[i].Enabled {
-			err := providers[i].markStartingSync(userCred)
+			err := providers[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "providers.markStartSync")
 			}
@@ -669,9 +684,9 @@ func (self *SCloudaccount) MarkEndSyncWithLock(ctx context.Context, userCred mcc
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
-	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
-		return nil
-	}
+	// if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
+	// 	return nil
+	// }
 
 	providers := self.GetCloudproviders()
 	for i := range providers {
@@ -820,7 +835,7 @@ func (self *SCloudaccount) importSubAccount(ctx context.Context, userCred mcclie
 				ownerId = userCred
 			} else {
 				// find default project of domain
-				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ownerId.GetProjectDomainId())
+				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ctx, ownerId.GetProjectDomainId())
 				if err != nil {
 					log.Errorf("cannot find a valid porject for domain %s", ownerId.GetProjectDomainId())
 					return nil, err
@@ -901,13 +916,13 @@ func (self *SCloudaccount) GetHostCount() (int, error) {
 
 func (self *SCloudaccount) GetVpcCount() (int, error) {
 	subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", self.Id).SubQuery()
-	q := VpcManager.Query().In("manager_id", subq)
+	q := VpcManager.Query().In("manager_id", subq).IsFalse("is_emulated")
 	return q.CountWithError()
 }
 
 func (self *SCloudaccount) GetStorageCount() (int, error) {
 	subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", self.Id).SubQuery()
-	q := StorageManager.Query().In("manager_id", subq)
+	q := StorageManager.Query().In("manager_id", subq).IsFalse("is_emulated")
 	return q.CountWithError()
 }
 
@@ -1481,6 +1496,18 @@ func (manager *SCloudaccountManager) initAllRecords() {
 	}
 }
 
+func (self *SCloudaccount) CanSync() bool {
+	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING || self.getSyncStatus2() == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING {
+		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second {
+			return true
+		} else {
+			return false
+		}
+	} else {
+		return true
+	}
+}
+
 func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 	if isStart && !options.Options.IsSlaveNode {
 		// mark all the records to be idle
@@ -1516,7 +1543,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	manager, err := account.getProviderInternal()
 	if err != nil {
 		log.Errorf("account.GetProvider failed: %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "account.getProviderInternal")
 	}
 	balance, status, err := manager.GetBalance()
 	if err != nil {
@@ -1534,7 +1561,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	sysInfo, err := manager.GetSysInfo()
 	if err != nil {
 		log.Errorf("manager.GetSysInfo fail %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "manager.GetSysInfo")
 	}
 	factory := manager.GetFactory()
 	diff, err := db.Update(account, func() error {
@@ -1587,7 +1614,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 	if err != nil {
 		account.markAllProvidersDicconnected(ctx, userCred)
 		account.markAccountDiscconected(ctx, userCred)
-		return err
+		return errors.Wrap(err, "account.probeAccountStatus")
 	}
 	account.markAccountConnected(ctx, userCred)
 	providers := account.importAllSubaccounts(ctx, userCred, subaccounts)
@@ -1596,7 +1623,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 			_, err := providers[i].prepareCloudproviderRegions(ctx, userCred)
 			if err != nil {
 				log.Errorf("syncCloudproviderRegion fail %s", err)
-				return err
+				return errors.Wrap(err, "providers[i].prepareCloudproviderRegions")
 			}
 		}
 	}
@@ -1615,13 +1642,39 @@ func (account *SCloudaccount) markAutoSync(userCred mcclient.TokenCredential) er
 	return nil
 }
 
+var (
+	cloudaccountPendingSyncs      = map[string]struct{}{}
+	cloudaccountPendingSyncsMutex = &sync.Mutex{}
+)
+
 func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCred mcclient.TokenCredential, waitChan chan error, autoSync bool) {
+	cloudaccountPendingSyncsMutex.Lock()
+	defer cloudaccountPendingSyncsMutex.Unlock()
+	if _, ok := cloudaccountPendingSyncs[account.Id]; ok {
+		if waitChan != nil {
+			go func() {
+				// an active cloudaccount sync task is running, return with conflict error
+				log.Errorf("an active cloudaccount sync task is running, early return with conflict error")
+				waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs")
+			}()
+		}
+		return
+	}
+	cloudaccountPendingSyncs[account.Id] = struct{}{}
+
 	RunSyncCloudAccountTask(func() {
+		func() {
+			cloudaccountPendingSyncsMutex.Lock()
+			defer cloudaccountPendingSyncsMutex.Unlock()
+			delete(cloudaccountPendingSyncs, account.Id)
+		}()
+
 		log.Debugf("syncAccountStatus %s %s", account.Id, account.Name)
 		err := account.syncAccountStatus(ctx, userCred)
 		if waitChan != nil {
 			if err != nil {
 				account.markEndSync(userCred)
+				err = errors.Wrap(err, "account.syncAccountStatus")
 			}
 			waitChan <- err
 		} else {
@@ -1631,7 +1684,8 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 				account.markAutoSync(userCred)
 				providers := account.GetEnabledCloudproviders()
 				for i := range providers {
-					providers[i].syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
+					provider := &providers[i]
+					provider.syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
 					syncCnt += 1
 				}
 			}

pkg/compute/models/cloudproviderregions.go

@@ -26,12 +26,14 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/nopanic"
 )
 
 type SCloudproviderregionManager struct {
@@ -220,17 +222,23 @@ func (manager *SCloudproviderregionManager) FetchByIdsOrCreate(providerId string
 	return cpr
 }
 
-func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	if !self.Enabled {
 		return fmt.Errorf("Cloudprovider(%s)region(%s) disabled", self.CloudproviderId, self.CloudregionId)
 	}
-	_, err := db.Update(self, func() error {
-		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
-		return nil
-	})
-	if err != nil {
-		log.Errorf("Failed to markStartingSync error: %v", err)
-		return err
+	regionIds := []string{}
+	if syncRange != nil {
+		regionIds, _ = syncRange.GetRegionIds()
+	}
+	if syncRange == nil || len(regionIds) == 0 || utils.IsInStringArray(self.CloudregionId, regionIds) {
+		_, err := db.Update(self, func() error {
+			self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
+			return nil
+		})
+		if err != nil {
+			log.Errorf("Failed to markStartingSync error: %v", err)
+			return err
+		}
 	}
 	return nil
 }
@@ -380,10 +388,12 @@ func (self *SCloudproviderregion) getSyncTaskKey() string {
 func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredential, syncRange SSyncRange, waitChan chan bool) {
 	self.markStartSync(userCred)
 	RunSyncCloudproviderRegionTask(self.getSyncTaskKey(), func() {
-		err := self.DoSync(context.Background(), userCred, syncRange)
-		if err != nil {
-			log.Errorf("DoSync faild %v", err)
-		}
+		nopanic.Run(func() {
+			err := self.DoSync(context.Background(), userCred, syncRange)
+			if err != nil {
+				log.Errorf("DoSync faild %v", err)
+			}
+		})
 		if waitChan != nil {
 			waitChan <- true
 		}

pkg/compute/models/cloudproviders.go

@@ -243,15 +243,15 @@ func (self *SCloudprovider) GetGuestCount() (int, error) {
 }
 
 func (self *SCloudprovider) GetHostCount() (int, error) {
-	return HostManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return HostManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getVpcCount() (int, error) {
-	return VpcManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return VpcManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getStorageCount() (int, error) {
-	return StorageManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return StorageManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getStoragecacheCount() (int, error) {
@@ -302,56 +302,60 @@ func (self *SCloudprovider) getPassword() (string, error) {
 	return utils.DescryptAESBase64(self.Id, self.Secret)
 }
 
-func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.TokenCredential) error {
-	if len(self.ProjectId) > 0 {
-		_, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
-		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("fetch existing tenant by id fail %s", err)
-			return errors.Wrap(err, "db.TenantCacheManager.FetchTenantById")
-		} else if err == nil {
-			return nil // find the project, skip sync
+func getTenant(ctx context.Context, projectId string, name string) (*db.STenant, error) {
+	if len(projectId) > 0 {
+		tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId)
+		if err != nil {
+			return nil, errors.Wrap(err, "TenantCacheManager.FetchTenantById")
 		}
+		return tenant, nil
 	}
-
-	if len(self.Name) == 0 {
-		log.Errorf("syncProject: provider name is empty???")
-		return errors.Error("cannot syncProject for empty name")
+	if len(name) == 0 {
+		return nil, errors.Error("cannot syncProject for empty name")
 	}
+	return db.TenantCacheManager.FetchTenantByName(ctx, name)
+}
 
-	tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, self.Name)
-	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("fetchTenantByIdorName error %s: %s", self.Name, err)
-		return errors.Wrap(err, "db.TenantCacheManager.FetchTenantByIdOrName")
+func createTenant(ctx context.Context, name, domainId, desc string) (string, string, error) {
+	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString(name), "generate_name")
+
+	params.Add(jsonutils.NewString(domainId), "domain_id")
+	params.Add(jsonutils.NewString(desc), "description")
+
+	resp, err := modules.Projects.Create(s, params)
+	if err != nil {
+		return "", "", errors.Wrap(err, "Projects.Create")
 	}
+	projectId, err := resp.GetString("id")
+	if err != nil {
+		return "", "", errors.Wrap(err, "resp.GetString")
+	}
+	return domainId, projectId, nil
+}
 
+func getOrCreateTenant(ctx context.Context, name, domainId, projectId, desc string) (string, string, error) {
+	tenant, err := getTenant(ctx, projectId, name)
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		return "", "", err
+	}
+	if err == sql.ErrNoRows || tenant.DomainId != domainId {
+		return createTenant(ctx, name, domainId, desc)
+	}
+	return tenant.DomainId, tenant.Id, nil
+}
+
+func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.TokenCredential) error {
 	account := self.GetCloudaccount()
 	if account == nil {
 		return errors.Error("no valid cloudaccount???")
 	}
 
-	var projectId, domainId string
-	if err == sql.ErrNoRows || tenant.DomainId != account.DomainId { // create one
-		s := auth.GetAdminSession(ctx, options.Options.Region, "")
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(self.Name), "generate_name")
-
-		domainId = account.DomainId
-		params.Add(jsonutils.NewString(domainId), "domain_id")
-		params.Add(jsonutils.NewString(fmt.Sprintf("auto create from cloud provider %s (%s)", self.Name, self.Id)), "description")
-
-		project, err := modules.Projects.Create(s, params)
-
-		if err != nil {
-			log.Errorf("create project fail %s", err)
-			return err
-		}
-		projectId, err = project.GetString("id")
-		if err != nil {
-			return err
-		}
-	} else {
-		domainId = tenant.DomainId
-		projectId = tenant.Id
+	desc := fmt.Sprintf("auto create from cloud provider %s (%s)", self.Name, self.Id)
+	domainId, projectId, err := getOrCreateTenant(ctx, self.Name, account.DomainId, self.ProjectId, desc)
+	if err != nil {
+		return errors.Wrap(err, "getOrCreateTenant")
 	}
 
 	return self.saveProject(userCred, domainId, projectId)
@@ -384,6 +388,44 @@ type SSyncRange struct {
 	Host   []string
 }
 
+func (sr *SSyncRange) GetRegionIds() ([]string, error) {
+	regionIds := []string{}
+	if len(sr.Host) == 0 && len(sr.Zone) == 0 && len(sr.Region) == 0 {
+		return regionIds, nil
+	}
+	hostQ := HostManager.Query().SubQuery()
+	hosts := hostQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(hostQ.Field("id"), sr.Host),
+		sqlchemy.In(hostQ.Field("name"), sr.Host),
+	)).SubQuery()
+	zoneQ := ZoneManager.Query().SubQuery()
+	zones := zoneQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(zoneQ.Field("id"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("name"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("id"), hosts.Query(hosts.Field("zone_id")).SubQuery()),
+	)).SubQuery()
+	regionQ := CloudregionManager.Query().SubQuery()
+	q := regionQ.Query(regionQ.Field("id")).Filter(sqlchemy.OR(
+		sqlchemy.In(regionQ.Field("id"), sr.Region),
+		sqlchemy.In(regionQ.Field("name"), sr.Region),
+		sqlchemy.In(regionQ.Field("id"), zones.Query(zones.Field("cloudregion_id")).SubQuery()),
+	))
+	rows, err := q.Rows()
+	if err != nil {
+		return nil, errors.Wrap(err, "q.Rows")
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var regionId string
+		err = rows.Scan(&regionId)
+		if err != nil {
+			return nil, errors.Wrap(err, "rows.Scan")
+		}
+		regionIds = append(regionIds, regionId)
+	}
+	return regionIds, nil
+}
+
 func (sr *SSyncRange) NeedSyncInfo() bool {
 	if sr.FullSync {
 		return true
@@ -539,7 +581,7 @@ func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context,
 		cloudaccount.markAutoSync(userCred)
 		cloudaccount.MarkSyncing(userCred)
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
@@ -606,7 +648,7 @@ func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred m
 	return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, DeepSync: true}, "")
 }
 
-func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
 		return nil
@@ -618,7 +660,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -627,7 +669,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	return nil
 }
 
-func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -639,7 +681,7 @@ func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) err
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -1136,9 +1178,10 @@ func (provider *SCloudprovider) GetCloudproviderRegions() []SCloudproviderregion
 func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange, wg *sync.WaitGroup, autoSync bool) {
 	provider.markSyncing(userCred)
 	cprs := provider.GetCloudproviderRegions()
+	regionIds, _ := syncRange.GetRegionIds()
 	syncCnt := 0
 	for i := range cprs {
-		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) {
+		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) && (len(regionIds) == 0 || utils.IsInStringArray(cprs[i].CloudregionId, regionIds)) {
 			syncCnt += 1
 			var waitChan chan bool = nil
 			if wg != nil {
@@ -1237,6 +1280,14 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	return nil
 }
 
+func (self *SCloudprovider) GetRegionDriver() (IRegionDriver, error) {
+	driver := GetRegionDriver(self.Provider)
+	if driver == nil {
+		return nil, fmt.Errorf("failed to found region driver for %s", self.Provider)
+	}
+	return driver, nil
+}
+
 func (self *SCloudprovider) ClearSchedDescCache() error {
 	hosts := make([]SHost, 0)
 	q := HostManager.Query().Equals("manager_id", self.Id)

pkg/compute/models/cloudregions.go

@@ -406,6 +406,9 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 	}
 
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
+		if !utils.IsInStringArray(self.Provider, api.PRIVATE_CLOUD_PROVIDERS) {
+			self.Name = cloudRegion.GetName()
+		}
 		self.Status = cloudRegion.GetStatus()
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()

pkg/compute/models/dbinstance_accounts.go

@@ -60,7 +60,8 @@ type SDBInstanceAccount struct {
 	db.SExternalizedResourceBase
 
 	// 数据库密码
-	Secret string `width:"256" charset:"ascii" nullable:"false" list:"domain" create:"optional"`
+	Secret string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+
 	// RDS实例Id
 	DBInstanceId string `width:"36" charset:"ascii" name:"dbinstance_id" nullable:"false" list:"user" create:"required" index:"true"`
 }
@@ -100,7 +101,7 @@ func (manager *SDBInstanceAccountManager) FetchOwnerId(ctx context.Context, data
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceAccountManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_databases.go

@@ -100,7 +100,7 @@ func (manager *SDBInstanceDatabaseManager) FetchOwnerId(ctx context.Context, dat
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceDatabaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_skus.go

@@ -59,7 +59,7 @@ type SDBInstanceSku struct {
 	SCloudregionResourceBase
 	Provider string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin"`
 
-	StorageType   string `list:"user" create:"optional"`
+	StorageType   string `width:"32" index:"true" list:"user" create:"optional"`
 	DiskSizeStep  int    `list:"user" default:"1" create:"optional"` //步长
 	MaxDiskSizeGb int    `list:"user" create:"optional"`
 	MinDiskSizeGb int    `list:"user" create:"optional"`
@@ -72,9 +72,9 @@ type SDBInstanceSku struct {
 	VcpuCount  int `nullable:"false" default:"1" list:"user" create:"optional"`
 	VmemSizeMb int `nullable:"false" list:"user" create:"required"`
 
-	Category      string `nullable:"false" list:"user" create:"optional"`
-	Engine        string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
-	EngineVersion string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	Category      string `width:"32" index:"true" nullable:"false" list:"user" create:"optional"`
+	Engine        string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	EngineVersion string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
 
 	Zone1  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`
 	Zone2  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`

pkg/compute/models/dbinstances.go

@@ -123,9 +123,6 @@ type SDBInstance struct {
 	Zone2 string `width:"36" charset:"ascii" nullable:"false" create:"optional" list:"user"`
 	// 可用区3
 	Zone3 string `width:"36" charset:"ascii" nullable:"false" create:"optional" list:"user"`
-
-	// 可用区Id(对应公有云的可用区Id)
-	ZoneId string `width:"36" charset:"ascii" nullable:"false" create:"optional"`
 }
 
 func (manager *SDBInstanceManager) GetContextManagers() [][]db.IModelManager {
@@ -383,14 +380,6 @@ func (self *SDBInstance) GetVpc() (*SVpc, error) {
 	return vpc.(*SVpc), nil
 }
 
-func (self *SDBInstance) GetZone() (*SZone, error) {
-	zone, err := ZoneManager.FetchById(self.ZoneId)
-	if err != nil {
-		return nil, err
-	}
-	return zone.(*SZone), nil
-}
-
 func (self *SDBInstance) GetNetwork() (*SNetwork, error) {
 	dbnet := DBInstanceNetworkManager.Query().SubQuery()
 	q := NetworkManager.Query()
@@ -1165,9 +1154,12 @@ func (self *SDBInstance) syncRemoveCloudDBInstance(ctx context.Context, userCred
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil { // cannot delete
-		return self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.RealDelete(ctx, userCred)
 }
@@ -1182,7 +1174,7 @@ func (self *SDBInstance) ValidateDeleteCondition(ctx context.Context) error {
 func (self *SDBInstance) GetDBInstanceSkuQuery() *sqlchemy.SQuery {
 	q := DBInstanceSkuManager.Query().Equals("storage_type", self.StorageType).Equals("category", self.Category).
 		Equals("cloudregion_id", self.CloudregionId).Equals("engine", self.Engine).Equals("engine_version", self.EngineVersion)
-	for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3, "zone_id": self.ZoneId} {
+	for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3} {
 		if len(v) > 0 {
 			q = q.Equals(k, v)
 		}
@@ -1285,6 +1277,30 @@ func (self *SDBInstance) SetZoneInfo(ctx context.Context, userCred mcclient.Toke
 	return err
 }
 
+func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) {
+	zone1 := extInstance.GetZone1Id()
+	if len(zone1) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone1)
+		if zone != nil {
+			self.Zone1 = zone.GetId()
+		}
+	}
+	zone2 := extInstance.GetZone2Id()
+	if len(zone2) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone2)
+		if zone != nil {
+			self.Zone2 = zone.GetId()
+		}
+	}
+	zone3 := extInstance.GetZone3Id()
+	if len(zone3) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone3)
+		if zone != nil {
+			self.Zone3 = zone.GetId()
+		}
+	}
+}
+
 func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extInstance cloudprovider.ICloudDBInstance) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Engine = extInstance.GetEngine()
@@ -1295,22 +1311,22 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 		self.DiskSizeGB = extInstance.GetDiskSizeGB()
 		self.StorageType = extInstance.GetStorageType()
 		self.Status = extInstance.GetStatus()
+		self.Port = extInstance.GetPort()
 
 		self.ConnectionStr = extInstance.GetConnectionStr()
 		self.InternalConnectionStr = extInstance.GetInternalConnectionStr()
 
 		self.MaintainTime = extInstance.GetMaintainTime()
-
-		self.ZoneId = extInstance.GetIZoneId()
-		err := self.setZoneInfo()
-		if err != nil {
-			log.Errorf("failed to set zone info for dbinstance %s(%s) error: %v", self.Name, self.Id, err)
-		}
+		self.SetZoneIds(extInstance)
 
 		if createdAt := extInstance.GetCreatedAt(); !createdAt.IsZero() {
 			self.CreatedAt = createdAt
 		}
 
+		if expiredAt := extInstance.GetExpiredAt(); !expiredAt.IsZero() {
+			self.ExpiredAt = expiredAt
+		}
+
 		factory, err := provider.GetProviderFactory()
 		if err != nil {
 			return errors.Wrap(err, "SyncWithCloudDBInstance.GetProviderFactory")
@@ -1368,11 +1384,7 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	instance.InternalConnectionStr = extInstance.GetInternalConnectionStr()
 
 	instance.MaintainTime = extInstance.GetMaintainTime()
-	instance.ZoneId = extInstance.GetIZoneId()
-	err = instance.setZoneInfo()
-	if err != nil {
-		log.Errorf("failed to set zone info for dbinstance %s error: %v", instance.ExternalId, err)
-	}
+	instance.SetZoneIds(extInstance)
 
 	if secgroupId := extInstance.GetSecurityGroupId(); len(secgroupId) > 0 {
 		q := SecurityGroupCacheManager.Query().Equals("manager_id", provider.Id).Equals("external_id", secgroupId)

pkg/compute/models/disks.go

@@ -416,7 +416,9 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc
 			input.Hypervisor,
 		)
 	} else {
-		diskConfig.Backend = api.STORAGE_LOCAL
+		if len(diskConfig.Backend) == 0 {
+			diskConfig.Backend = api.STORAGE_LOCAL
+		}
 		serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor)
 		if err != nil {
 			return nil, err
@@ -830,6 +832,9 @@ func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredenti
 	}
 	addDisk := sizeMb - disk.DiskSize
 	storage := disk.GetStorage()
+	if storage == nil {
+		return httperrors.NewInternalServerError("disk has no valid storage")
+	}
 	if host := storage.GetMasterHost(); host != nil {
 		if err := host.GetHostDriver().ValidateDiskSize(storage, sizeMb>>10); err != nil {
 			return httperrors.NewInputParameterError(err.Error())
@@ -986,6 +991,10 @@ func (self *SDisk) ValidatePurgeCondition(ctx context.Context) error {
 func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) error {
 	if !isPurge {
 		storage := self.GetStorage()
+		if storage == nil {
+			// storage is empty, a dirty data, allow delete
+			return nil
+		}
 		host := storage.GetMasterHost()
 		if host == nil {
 			return httperrors.NewBadRequestError("storage of disk no valid host")
@@ -1098,6 +1107,9 @@ func (self *SDisk) GetPathAtHost(host *SHost) string {
 
 func (self *SDisk) GetFetchUrl() string {
 	storage := self.GetStorage()
+	if storage == nil {
+		return ""
+	}
 	host := storage.GetMasterHost()
 	return fmt.Sprintf("%s/disks/%s", host.GetFetchUrl(true), self.Id)
 }
@@ -1126,7 +1138,10 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien
 	diskObj, err := db.FetchByExternalId(manager, vdisk.GetGlobalId())
 	if err != nil {
 		if err == sql.ErrNoRows {
-			vstorage, _ := vdisk.GetIStorage()
+			vstorage, err := vdisk.GetIStorage()
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to GetIStorage of vdisk %q", vdisk.GetName())
+			}
 
 			storageObj, err := db.FetchByExternalId(StorageManager, vstorage.GetGlobalId())
 			if err != nil {
@@ -1323,7 +1338,6 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	}
 	extDisk.Refresh()
 
-	storage := self.GetStorage()
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		// self.Name = extDisk.GetName()
 		self.Status = extDisk.GetStatus()
@@ -1350,8 +1364,12 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 		self.IsEmulated = extDisk.IsEmulated()
 
 		if provider.GetFactory().IsSupportPrepaidResources() && !recycle {
-			self.BillingType = extDisk.GetBillingType()
-			self.ExpiredAt = extDisk.GetExpiredAt()
+			if billintType := extDisk.GetBillingType(); len(billintType) > 0 {
+				self.BillingType = extDisk.GetBillingType()
+			}
+			if expiredAt := extDisk.GetExpiredAt(); !expiredAt.IsZero() {
+				self.ExpiredAt = extDisk.GetExpiredAt()
+			}
 		}
 
 		if createdAt := extDisk.GetCreatedAt(); !createdAt.IsZero() {
@@ -1370,6 +1388,10 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	if err != nil {
 		return errors.Wrapf(err, "Get snapshot policies of ICloudDisk %s.", extDisk.GetId())
 	}
+	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	err = SnapshotPolicyDiskManager.SyncByDisk(ctx, userCred, snapshotpolicies, syncOwnerId, self, storage)
 	if err != nil {
 		return err
@@ -1527,7 +1549,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 	// 	diskConfig.SizeMb = options.Options.DefaultDiskSize // MB
 	// else
 	if len(info.ImageId) == 0 && info.SizeMb == 0 {
-		return nil, httperrors.NewInputParameterError("Diskinfo not contains either imageID or size")
+		return nil, httperrors.NewInputParameterError("Diskinfo index %d: both imageID and size are absent", info.Index)
 	}
 	return info, nil
 }
@@ -1802,6 +1824,8 @@ func (self *SDisk) getMoreDetails(ctx context.Context, userCred mcclient.TokenCr
 		policy := api.SimpleSnapshotPolicy{}
 		policy.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sps[i].RepeatWeekdays)
 		policy.TimePoints = SnapshotPolicyManager.TimePointsToIntArray(sps[i].TimePoints)
+		policy.Id = sps[i].Id
+		policy.Name = sps[i].Name
 		out.Snapshotpolicies = append(out.Snapshotpolicies, policy)
 	}
 	storage := self.GetStorage()
@@ -1903,6 +1927,9 @@ func (self *SDisk) SwitchToBackup(userCred mcclient.TokenCredential) error {
 
 func (self *SDisk) ClearHostSchedCache() error {
 	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	hosts := storage.GetAllAttachingHosts()
 	if hosts == nil {
 		return fmt.Errorf("get attaching host error")
@@ -2062,7 +2089,11 @@ func (disk *SDisk) validateDiskAutoCreateSnapshot() error {
 	if len(guests) == 0 {
 		return fmt.Errorf("Disks %s not attach guest, can't create snapshot", disk.GetName())
 	}
-	if len(guests) == 1 && utils.IsInStringArray(disk.GetStorage().StorageType, api.FIEL_STORAGE) {
+	storage := disk.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
+	if len(guests) == 1 && utils.IsInStringArray(storage.StorageType, api.FIEL_STORAGE) {
 		if !utils.IsInStringArray(guests[0].Status, []string{api.VM_RUNNING, api.VM_READY}) {
 			return fmt.Errorf("Guest(%s) in status(%s) cannot do disk snapshot", guests[0].Id, guests[0].Status)
 		}
@@ -2243,6 +2274,9 @@ func (self *SDisk) GetDynamicConditionInput() *jsonutils.JSONDict {
 
 func (self *SDisk) IsNeedWaitSnapshotsDeleted() (bool, error) {
 	storage := self.GetStorage()
+	if storage == nil {
+		return false, fmt.Errorf("no valid storage")
+	}
 	if storage.StorageType == api.STORAGE_RBD {
 		scnt, err := self.GetSnapshotCount()
 		if err != nil {
@@ -2309,7 +2343,12 @@ func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCre
 	}
 	provider := self.GetCloudprovider()
 	syncOwnerId := provider.GetOwnerId()
-	region := self.GetStorage().GetRegion()
+	storage := self.GetStorage()
+	if storage == nil {
+		syncResult.Error(fmt.Errorf("no valid storage"))
+		return syncResult
+	}
+	region := storage.GetRegion()
 
 	extSnapshots, err := extDisk.GetISnapshots()
 	if err != nil {

pkg/compute/models/dnsrecords.go

@@ -37,6 +37,8 @@ type SDnsRecordManager struct {
 	db.SAdminSharableVirtualResourceBaseManager
 }
 
+var _ db.IAdminSharableVirtualModelManager = DnsRecordManager
+
 var DnsRecordManager *SDnsRecordManager
 
 func init() {
@@ -276,35 +278,38 @@ func (man *SDnsRecordManager) checkRecordValue(typ, val string) error {
 
 func (man *SDnsRecordManager) validateModelData(
 	ctx context.Context,
-	userCred mcclient.TokenCredential,
-	ownerId mcclient.IIdentityProvider,
-	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
-) (*jsonutils.JSONDict, error) {
-	records, err := man.ParseInputInfo(data)
+	isCreate bool,
+) (records []string, err error) {
+	data.Remove("records")
+	records, err = man.ParseInputInfo(data)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if len(records) == 0 {
-		return nil, httperrors.NewInputParameterError("Empty record")
+		if isCreate {
+			err = httperrors.NewInputParameterError("Empty record")
+			return
+		}
+		return
 	}
 	recType := man.getRecordsType(records)
 	name, err := data.GetString("name")
 	if err != nil {
-		return nil, err
+		return
 	}
 	err = man.checkRecordName(recType, name)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if data.Contains("ttl") {
-		jo, err := data.Get("ttl")
+		var (
+			ttl int64
+		)
+		ttl, err = data.Int("ttl")
 		if err != nil {
-			return nil, err
-		}
-		ttl, err := jo.Int()
-		if err != nil {
-			return nil, httperrors.NewInputParameterError("invalid ttl: %s", err)
+			err = httperrors.NewInputParameterError("invalid ttl: %s", err)
+			return
 		}
 		if ttl == 0 {
 			// - Create: use the database default
@@ -312,10 +317,11 @@ func (man *SDnsRecordManager) validateModelData(
 			data.Remove("ttl")
 		} else if ttl < 0 || ttl > 0x7fffffff {
 			// positive values of a signed 32 bit number.
-			return nil, httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			err = httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			return
 		}
 	}
-	return data, err
+	return records, nil
 }
 
 func (man *SDnsRecordManager) ValidateCreateData(
@@ -325,7 +331,7 @@ func (man *SDnsRecordManager) ValidateCreateData(
 	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
-	data, err := man.validateModelData(ctx, userCred, ownerId, query, data)
+	_, err := man.validateModelData(ctx, data, true)
 	if err != nil {
 		return nil, err
 	}
@@ -402,15 +408,11 @@ func (rec *SDnsRecord) GetInfo() []string {
 
 func (rec *SDnsRecord) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	data.UpdateDefault(jsonutils.Marshal(rec))
-	data, err := DnsRecordManager.validateModelData(ctx, userCred, rec.GetOwnerId(), query, data)
+	records, err := DnsRecordManager.validateModelData(ctx, data, false)
 	if err != nil {
 		return nil, err
 	}
-	{
-		records, err := DnsRecordManager.ParseInputInfo(data)
-		if err != nil {
-			return nil, err
-		}
+	if len(records) > 0 {
 		data.Set("records", jsonutils.NewString(strings.Join(records, DNS_RECORDS_SEPARATOR)))
 	}
 	return rec.SAdminSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)

pkg/compute/models/elasticcache_instances.go

@@ -116,7 +116,7 @@ func elasticcacheSubResourceFetchOwnerId(ctx context.Context, data jsonutils.JSO
 		return ec.(*SElasticcache).GetOwnerId(), nil
 	}
 
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 // elastic cache 子资源获取owner query
@@ -325,9 +325,12 @@ func (self *SElasticcache) syncRemoveCloudElasticcache(ctx context.Context, user
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.Delete(ctx, userCred)
 }

pkg/compute/models/external_projects.go

@@ -182,8 +182,10 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
-		self.ProjectId = provider.ProjectId
-		self.DomainId = provider.DomainId
+		if self.DomainId != provider.DomainId {
+			self.ProjectId = provider.ProjectId
+			self.DomainId = provider.DomainId
+		}
 		return nil
 	})
 	if err != nil {
@@ -198,18 +200,25 @@ func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context,
 	project := SExternalProject{}
 	project.SetModelManager(manager, &project)
 
-	newName, err := db.GenerateName(manager, userCred, extProject.GetName())
-	if err != nil {
-		return nil, err
-	}
-	project.Name = newName
+	project.Name = extProject.GetName()
 	project.ExternalId = extProject.GetGlobalId()
 	project.IsEmulated = extProject.IsEmulated()
 	project.ManagerId = provider.Id
 	project.DomainId = provider.DomainId
 	project.ProjectId = provider.ProjectId
+	account := provider.GetCloudaccount()
+	if account != nil && account.AutoCreateProject {
+		desc := fmt.Sprintf("auto create from cloud project %s (%s)", project.Name, project.ExternalId)
+		domainId, projectId, err := getOrCreateTenant(ctx, project.Name, provider.DomainId, "", desc)
+		if err != nil {
+			log.Errorf("failed to get or create tenant %s(%s) %v", project.Name, project.ExternalId, err)
+		} else {
+			project.DomainId = domainId
+			project.ProjectId = projectId
+		}
+	}
 
-	err = manager.TableSpec().Insert(&project)
+	err := manager.TableSpec().Insert(&project)
 	if err != nil {
 		log.Errorf("newFromCloudProject fail %s", err)
 		return nil, err

pkg/compute/models/guest_actions.go

@@ -3991,6 +3991,20 @@ func (self *SGuest) PerformSyncFixNics(ctx context.Context,
 		return nil, httperrors.NewInputParameterError("missing field ip, list of ip")
 	}
 	iplist := iplistArray.(*jsonutils.JSONArray).GetStringArray()
+	errs := make([]error, 0)
+	for i := range vnics {
+		ip := vnics[i].GetIP()
+		if len(ip) == 0 {
+			continue
+		}
+		_, err := host.getNetworkOfIPOnHost(ip)
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, ip))
+		}
+	}
+	if len(errs) > 0 {
+		return nil, httperrors.NewInvalidStatusError(errors.NewAggregate(errs).Error())
+	}
 	result := self.SyncVMNics(ctx, userCred, host, vnics, iplist)
 	if result.IsError() {
 		return nil, httperrors.NewInternalServerError(result.Result())
@@ -4461,6 +4475,9 @@ func (self *SGuest) PerformSnapshotAndClone(
 		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
+	} else {
+		cancelRegionUsage := &SRegionQuota{Snapshot: snapshotUsage.Snapshot}
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, cancelRegionUsage, true)
 	}
 
 	err = self.StartInstanceSnapshotAndCloneTask(

pkg/compute/models/guest_template.go

@@ -246,8 +246,8 @@ func (gt *SGuestTemplate) getMoreDetails(ctx context.Context, userCred mcclient.
 	configInfo.Disks = disks
 
 	// keypair
-	if len(input.KeypairId) > 0 {
-		model, err := KeypairManager.FetchById(input.KeypairId)
+	if len(input.Keypair) > 0 {
+		model, err := KeypairManager.FetchByIdOrName(userCred, input.Keypair)
 		if err == nil {
 			keypair := model.(*SKeypair)
 			configInfo.Keypair = keypair.GetName()
@@ -341,7 +341,7 @@ func (gt *SGuestTemplate) PerformPublic(ctx context.Context, userCred mcclient.T
 
 	// check for below private resource in the guest template
 	privateResource := map[string]int{
-		"keypair":           len(input.KeypairId),
+		"keypair":           len(input.Keypair),
 		"instance group":    len(input.InstanceGroupIds),
 		"instance snapshot": len(input.InstanceSnapshotId),
 	}

pkg/compute/models/guestdisks.go

@@ -178,15 +178,11 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 			desc.Add(jsonutils.NewString(storagecacheimg.Path), "image_path")
 		}
 	}
-	storage := disk.GetStorage()
-	// XXX ???
 	if host.HostType == api.HOST_TYPE_HYPERVISOR {
 		desc.Add(jsonutils.NewString(disk.StorageId), "storage_id")
 		localpath := disk.GetPathAtHost(host)
 		if len(localpath) == 0 {
 			desc.Add(jsonutils.JSONTrue, "migrating")
-			target := host.GetLeastUsedStorage(storage.StorageType)
-			desc.Add(jsonutils.NewString(target.Id), "target_storage_id")
 			disk.SetStatus(nil, api.DISK_START_MIGRATE, "migration")
 		} else {
 			desc.Add(jsonutils.NewString(localpath), "path")