Merge pull request #8223 from yousong/automated-cherry-pick-of-#8222-upstream-release-3.0

Automated cherry pick of #8222: webconsole: ssh: each argument on its own line

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/logger /opt/yunion/bin/logger
 

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -7,7 +7,7 @@ ENV TZ Asia/Shanghai
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates && \
+    apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.region-dns

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/region-dns /opt/yunion/bin/region-dns

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.yunionconf

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/yunionconf /opt/yunion/bin/yunionconf
 

cmd/climc/shell/cloudaccounts.go

@@ -26,6 +26,28 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
+func parseGcpCredential(filename string) (jsonutils.JSONObject, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	authParams, err := jsonutils.Parse(data)
+	if err != nil {
+		return nil, err
+	}
+	ret := jsonutils.NewDict()
+	for _, k := range []string{
+		"client_email",
+		"project_id",
+		"private_key_id",
+		"private_key",
+	} {
+		v, _ := authParams.Get(k)
+		ret.Add(v, fmt.Sprintf("gcp_%s", k))
+	}
+	return ret, nil
+}
+
 func init() {
 
 	type CloudaccountListOptions struct {
@@ -134,11 +156,7 @@ func init() {
 	R(&options.SGoogleCloudAccountCreateOptions{}, "cloud-account-create-google", "Create a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountCreateOptions) error {
 		params := jsonutils.Marshal(args)
 		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Google"), "provider")
-		data, err := ioutil.ReadFile(args.GoogleJsonFile)
-		if err != nil {
-			return err
-		}
-		authParams, err := jsonutils.Parse(data)
+		authParams, err := parseGcpCredential(args.GoogleJsonFile)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/event.go

@@ -0,0 +1,9 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initEvent() {
+	initK8sNamespaceResource("event", k8s.Events)
+}

cmd/climc/shell/k8s/k8s.go

@@ -16,6 +16,11 @@ package k8s
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+
+	"github.com/ghodss/yaml"
 
 	"yunion.io/x/jsonutils"
 
@@ -57,7 +62,7 @@ func init() {
 	initPVC()
 	initJob()
 	initCronJob()
-
+	initEvent()
 	initRbac()
 
 	initApp()
@@ -217,6 +222,71 @@ func NewK8sNsResourceGetCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cm
 	)
 }
 
+func NewK8sNsResourceGetRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("show-raw"),
+		fmt.Sprintf("Show k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			ret, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			printObjectYAML(ret)
+			return nil
+		},
+	)
+}
+
+func NewK8sResourceEditRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("edit-raw"),
+		fmt.Sprintf("Edit and update k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			rawData, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			yamlBytes := rawData.YAMLString()
+			tempfile, err := ioutil.TempFile("", fmt.Sprintf("k8s-%s-%s*.yaml", cmdN.Kind, args.NAME))
+			if err != nil {
+				return err
+			}
+			defer os.Remove(tempfile.Name())
+			if _, err := tempfile.Write([]byte(yamlBytes)); err != nil {
+				return err
+			}
+			if err := tempfile.Close(); err != nil {
+				return err
+			}
+
+			cmd := exec.Command("vim", tempfile.Name())
+			cmd.Stdin = os.Stdin
+			cmd.Stdout = os.Stdout
+			if err := cmd.Run(); err != nil {
+				return err
+			}
+			content, err := ioutil.ReadFile(tempfile.Name())
+			if err != nil {
+				return err
+			}
+			jsonBytes, err := yaml.YAMLToJSON(content)
+			if err != nil {
+				return err
+			}
+			body, err := jsonutils.Parse(jsonBytes)
+			if err != nil {
+				return err
+			}
+			if _, err := manager.UpdateRaw(s, args.NAME, args.Params(), body.(*jsonutils.JSONDict)); err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+}
+
 func NewK8sResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cmd {
 	return NewCommand(
 		&o.ResourceDeleteOptions{},
@@ -244,11 +314,13 @@ func NewK8sNsResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager)
 	return deleteCmd
 }
 
-func initK8sNamespaceResource(kind string, manager modulebase.Manager) *ShellCommands {
+func initK8sNamespaceResource(kind string, manager k8s.IClusterResourceManager) *ShellCommands {
 	cmdN := NewCmdNameFactory(kind)
 	return NewShellCommands(cmdN.Do).AddR(
 		NewK8sNsResourceListCmd(cmdN, manager),
 		NewK8sNsResourceGetCmd(cmdN, manager),
 		NewK8sNsResourceDeleteCmd(cmdN, manager),
+		NewK8sNsResourceGetRawCmd(cmdN, manager),
+		NewK8sResourceEditRawCmd(cmdN, manager),
 	)
 }

cmd/climc/shell/policies.go

@@ -134,7 +134,7 @@ func init() {
 		return nil
 	}
 	R(&PolicyPatchOptions{}, "policy-patch", "Patch policy", updateFunc)
-	R(&PolicyPatchOptions{}, "policy-update", "Patch policy", updateFunc)
+	R(&PolicyPatchOptions{}, "policy-update", "Update policy", updateFunc)
 
 	type PolicyPerformOptions struct {
 		ID string `help:"ID of policy to update"`
@@ -285,12 +285,30 @@ func init() {
 		if err != nil {
 			log.Fatalf("Set log level %q: %v", "debug", err)
 		}
+		if args.Debug {
+			rbacutils.ShowMatchRuleDebug = true
+		}
 		auth.InitFromClientSession(s)
 		policy.EnableGlobalRbac(15*time.Second, 15*time.Second, false)
 		if args.Debug {
 			consts.EnableRbacDebug()
 		}
 
+		findPolicy := false
+		for !findPolicy {
+			all := policy.PolicyManager.AllPolicies()
+			for _, allP := range all {
+				if len(allP) > 0 {
+					findPolicy = true
+					break
+				}
+			}
+			if findPolicy {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
 		req := jsonutils.NewDict()
 		for i := 0; i < len(args.Request); i += 1 {
 			parts := strings.Split(args.Request[i], ":")
@@ -356,6 +374,7 @@ func init() {
 				Context: mcclient.SAuthContext{
 					Ip: args.Ip,
 				},
+				Token: "faketoken",
 			}
 		} else {
 			token = s.GetToken()
@@ -367,6 +386,10 @@ func init() {
 		}
 		printObject(result)
 
+		for _, r := range args.Role {
+			fmt.Println("role", r, "matched policies:", policy.PolicyManager.RoleMatchPolicies(r))
+		}
+
 		fmt.Println("userCred:", token)
 		for _, scope := range []rbacutils.TRbacScope{
 			rbacutils.ScopeSystem,
@@ -375,7 +398,7 @@ func init() {
 			rbacutils.ScopeUser,
 			rbacutils.ScopeNone,
 		} {
-			m := policy.PolicyManager.MatchedPolicies(scope, token)
+			m := policy.PolicyManager.MatchedPolicyNames(scope, token)
 			fmt.Println("matched", scope, "policies:", m)
 		}
 

cmd/climc/shell/servers.go

@@ -987,6 +987,7 @@ func init() {
 				config.Hosts[i].HostIp = yamlConfig.Hosts[i].HostIp
 				config.Hosts[i].XmlFilePath = yamlConfig.Hosts[i].XmlFilePath
 				config.Hosts[i].Servers = make([]compute.SLibvirtServerConfig, len(yamlConfig.Hosts[i].Servers))
+				config.Hosts[i].MonitorPath = yamlConfig.Hosts[i].MonitorPath
 				for j := 0; j < len(yamlConfig.Hosts[i].Servers); j++ {
 					config.Hosts[i].Servers[j].MacIp = make(map[string]string)
 					mac := yamlConfig.Hosts[i].Servers[j].Mac
@@ -1009,11 +1010,11 @@ func init() {
 		if err != nil {
 			return err
 		}
-		for i := 0; i < len(params); i++ {
-			val := jsonutils.NewDict()
-			val.Set(modules.Servers.KeywordPlural, params[i])
-			params[i] = val
-		}
+		//for i := 0; i < len(params); i++ {
+		//	val := jsonutils.NewDict()
+		//	val.Set(modules.Servers.KeywordPlural, params[i])
+		//	params[i] = val
+		//}
 
 		results := modules.Servers.BatchPerformClassAction(s, "import-from-libvirt", params)
 		printBatchResults(results, modules.Servers.GetColumns(s))

go.mod

@@ -3,7 +3,7 @@ module yunion.io/x/onecloud
 go 1.12
 
 require (
-	cloud.google.com/go v0.38.0
+	cloud.google.com/go v0.38.0 // indirect
 	github.com/360EntSecGroup-Skylar/excelize v1.4.0
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible
 	github.com/Azure/go-autorest v10.15.5+incompatible
@@ -122,7 +122,7 @@ require (
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191008105621-543471e840be
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
-	google.golang.org/api v0.13.0 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
 	google.golang.org/grpc v1.23.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
@@ -138,12 +138,12 @@ require (
 	k8s.io/klog v0.1.0 // indirect
 	k8s.io/kubernetes v1.12.3
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
-	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
+	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
+	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
+	yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
-	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
+	yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e
 )
 
 replace github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 => github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859

go.sum

@@ -226,8 +226,6 @@ github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googollee/go-engine.io v0.0.0-20180829091931-e2f255711dcb h1:n22Aukg/TjoypWc37dbKIpCsz0VMFPD36HQk1WKvg3A=
@@ -263,8 +261,6 @@ github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead h1:Y9WOGZY2nw5ksbEf5A
 github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
@@ -539,7 +535,6 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -553,7 +548,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -591,7 +585,6 @@ golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190830023255-19e00faab6ad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -615,7 +608,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.zx2c4.com/wireguard v0.0.20190908 h1:SUoXDdwSMtomLdvke+zz83/u9tNvl4hHmcTIWp38tow=
@@ -625,8 +617,6 @@ golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987/go.mod h1:7
 google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -642,7 +632,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -676,7 +665,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.0.0-20181004124137-fd83cbc87e76 h1:cGc6jt7tNK7a2WfgNKjxjoU/UXXr9Q7JTqvCupZ+6+Y=
 k8s.io/api v0.0.0-20181004124137-fd83cbc87e76/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
@@ -694,21 +682,23 @@ yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051 h1:vtZw2iwGrsARNSwRTREGjmr2BWPdxbmXVkb3kI1qu28=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda h1:wSwQj3MDGchGYM2RAo1riYhrTz4apH+5XyBnchuDi84=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501 h1:i1r9XvbdxH3FgTCLmTaRi3MzQqhiQimXJRlUOPgrxnU=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b h1:Z9z+7iegu0HXuL+S8taVWRd1P4b9JJOgPXIeoqYrj7c=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRiE7QcPG8eTp0GmsHbg=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa h1:VizPfW8+mLFEE7W/97zxQJbfdxchi2dn1M/Y7YBwTTc=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e h1:pctCe/EPel3F1B83pJ2q9b34Umd1NdbLW1Yd+Lzur2s=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -29,6 +29,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apigateway/constants"
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	policytool "yunion.io/x/onecloud/pkg/apigateway/policy"
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
@@ -360,7 +361,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
+		httperrors.ForbiddenError(w, "user forbidden login from web")
 		return false
 	}
 	return true
@@ -569,6 +570,7 @@ func (this *projectRoles) add(roleId, roleName string) {
 
 func (this *projectRoles) getToken(scope rbacutils.TRbacScope, user, userId, domain, domainId string, ip string) mcclient.TokenCredential {
 	return &mcclient.SSimpleToken{
+		Token:           "faketoken",
 		Domain:          domain,
 		DomainId:        domainId,
 		User:            user,
@@ -608,12 +610,12 @@ func (this *projectRoles) json(user, userId, domain, domainId string, ip string)
 	}
 	obj.Add(roles, "roles")
 	for _, scope := range []rbacutils.TRbacScope{
-		rbacutils.ScopeSystem,
-		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeSystem,
 	} {
 		token := this.getToken(scope, user, userId, domain, domainId, ip)
-		matches := policy.PolicyManager.MatchedPolicies(scope, token)
+		matches := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		obj.Add(jsonutils.NewStringArray(matches), fmt.Sprintf("%s_policies", scope))
 		if len(matches) > 0 {
 			obj.Add(jsonutils.JSONTrue, fmt.Sprintf("%s_capable", scope))
@@ -646,8 +648,11 @@ func getUserAuthCookie(ctx context.Context, s *mcclient.ClientSession, token mcc
 }
 
 func getLBAgentInfo(s *mcclient.ClientSession, token mcclient.TokenCredential) (*jsonutils.JSONDict, error) {
-
-	lbagents, err := modules.LoadbalancerAgents.List(s, nil)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString("hb_last_seen.isnotempty()"), "filter.0")
+	params.Add(jsonutils.NewInt(1), "limit")
+	params.Add(jsonutils.JSONFalse, "details")
+	lbagents, err := modules.LoadbalancerAgents.List(s, params)
 	if err != nil {
 		return nil, errors.Wrapf(err, "user %s get lbagent", token.GetUserName())
 	}
@@ -665,6 +670,7 @@ func getLBAgentInfo(s *mcclient.ClientSession, token mcclient.TokenCredential) (
 }
 
 func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.TokenCredential, req *http.Request) (*jsonutils.JSONDict, error) {
+	log.Infof("getUserInfo modules.UsersV3.Get")
 	usr, err := modules.UsersV3.Get(s, token.GetUserId(), nil)
 	if err != nil {
 		log.Errorf("modules.UsersV3.Get fail %s", err)
@@ -693,6 +699,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 	data.Add(jsonutils.NewString(token.GetProjectDomain()), "projectDomain")
 	data.Add(jsonutils.NewString(token.GetProjectDomainId()), "projectDomainId")
 
+	log.Infof("getUserInfo modules.RoleAssignments.List")
 	query := jsonutils.NewDict()
 	query.Add(jsonutils.JSONNull, "effective")
 	query.Add(jsonutils.JSONNull, "include_names")
@@ -735,7 +742,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
 	} {
-		p := policy.PolicyManager.MatchedPolicies(scope, token)
+		p := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		data.Add(jsonutils.NewStringArray(p), fmt.Sprintf("%s_policies", scope))
 		if scope == rbacutils.ScopeSystem {
 			data.Add(jsonutils.NewStringArray(p), "admin_policies")
@@ -772,6 +779,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		log.Errorf("fail to find services????: %#v %s", adminToken, curReg)
 	}
 
+	log.Infof("getUserInfo getLBAgentInfo")
 	lb, err := getLBAgentInfo(s, adminToken)
 	if err != nil {
 		log.Errorf("getLBAgentInfo fail %s", err)
@@ -788,13 +796,23 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		}
 	}
 
+	log.Infof("getUserInfo modules.Hosts.Get")
 	s2 := auth.GetSession(ctx, token, FetchRegion(req), "v2")
-	cap, err := modules.Capabilities.List(s2, nil)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString("host_type"), "field")
+	params.Add(jsonutils.NewString("system"), "scope")
+	params.Add(jsonutils.JSONTrue, "usable")
+	params.Add(jsonutils.JSONTrue, "show_emulated")
+	cap, err := modules.Hosts.Get(s2, "distinct-field", params)
 	if err != nil {
-		log.Errorf("modules.Capabilities.List fail %s", err)
+		log.Errorf("modules.Servers.Get distinct-field fail %s", err)
 	} else {
-		hypervisors, _ := cap.Data[0].Get("hypervisors")
-		data.Add(hypervisors, "hypervisors")
+		hostTypes, _ := jsonutils.GetStringArray(cap, "host_type")
+		hypervisors := make([]string, len(hostTypes))
+		for i, hostType := range hostTypes {
+			hypervisors[i] = compute.HOSTTYPE_HYPERVISOR[hostType]
+		}
+		data.Add(jsonutils.NewStringArray(hypervisors), "hypervisors")
 	}
 
 	data.Add(menus, "menus")

pkg/apigateway/handler/init.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
+)

pkg/apigateway/handler/misc.go

@@ -28,6 +28,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -171,19 +173,23 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	}
 
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +205,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 

pkg/apigateway/handler/resource.go

@@ -606,28 +606,45 @@ func (f *ResourceHandlers) patchJointHandler(ctx context.Context, w http.Respons
 	}
 }
 
-// batch update Joint
+// * batch update Joint
+// * put specific
+// /<resname>/<resid>/<resname2>
+// /<resname>/<resid>/<spec>
 func (f *ResourceHandlers) batchUpdateJointHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	req := newRequest(ctx, w, r).WithMod1().WithMod2()
+	req := newRequest(ctx, w, r).WithMod1()
 	if err := req.Error(); err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
 	}
 	session := req.Session()
 	module := req.Mod1()
-	module2 := req.Mod2()
 	body := req.Body()
 	query := req.Query()
 
-	idlist := fetchIdList(query, w)
-	if idlist == nil {
+	if idlist, _ := query.GetArray("id"); len(idlist) == 0 {
+		// do put specific
+		spec := req.ResName2()
+		obj, e := module.PutSpecific(session, req.ResID(), spec, query, body)
+		if e != nil {
+			httperrors.GeneralServerError(w, e)
+		} else {
+			appsrv.SendJSON(w, obj)
+		}
 		return
 	}
+
+	req = req.WithMod2()
+	if err := req.Error(); err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	module2 := req.Mod2()
 	jmod, e := modulebase.GetJointModule2(session, module, module2)
 	if e != nil { // update joint
 		httperrors.GeneralServerError(w, e)
 		return
 	}
+	idlist := fetchIdList(query, w)
 	ret := jmod.BatchUpdate(session, req.ResID(), idlist, query, body)
 	w.WriteHeader(207)
 	appsrv.SendJSON(w, modulebase.SubmitResults2JSON(ret))

pkg/apis/compute/cloudregions_const.go

@@ -56,6 +56,7 @@ const (
 	CITY_NAN_JING      = "Nanjing"     //南京
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
+	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京
@@ -111,6 +112,7 @@ const (
 	CITY_US_GOV_WEST    = "us-gov-west"    //???
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
+	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -149,6 +151,12 @@ const (
 	// 巴西
 	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
 
+	// 智利
+	CITY_SANTIAGO = "Santiago" // 圣地亚哥
+
+	// 墨西哥
+	CITY_MEXICO = "Mexico" // 墨西哥
+
 	// 荷兰
 	CITY_HOLLAND = "Holland" //荷兰
 
@@ -194,4 +202,6 @@ const (
 	COUNTRY_CODE_VN = "VN" //越南
 	COUNTRY_CODE_CH = "CH" //瑞士
 	COUNTRY_CODE_NO = "NO" //挪威
+	COUNTRY_CODE_MX = "MX" //墨西哥
+	COUNTRY_CODE_CL = "CL" //智利
 )

pkg/apis/identity/assignments.go

@@ -62,3 +62,7 @@ func (ra *SRoleAssignment) GetRoles() []string {
 func (ra *SRoleAssignment) GetLoginIp() string {
 	return ""
 }
+
+func (ra *SRoleAssignment) GetTokenString() string {
+	return "faketoken"
+}

pkg/apis/identity/consts.go

@@ -144,6 +144,10 @@ var (
 			"etcd_cacert",
 			"etcd_cert",
 			"etcd_key",
+
+			"bootstrap_admin_user_password",
+			"reset_admin_user_password",
+			"fernet_key_repository",
 		},
 	}
 )

pkg/appsrv/ring.go

@@ -25,10 +25,12 @@ type Ring struct {
 }
 
 func NewRing(size int) *Ring {
-	r := Ring{buffer: make([]interface{}, size+1),
+	r := Ring{
+		buffer: make([]interface{}, size+1),
 		header: 0,
 		tail:   0,
-		lock:   &sync.Mutex{}}
+		lock:   &sync.Mutex{},
+	}
 	return &r
 }
 
@@ -58,6 +60,7 @@ func (r *Ring) Pop() interface{} {
 		return nil
 	}
 	ret := r.buffer[r.tail]
+	r.buffer[r.tail] = nil
 	r.tail = nextPointer(r.tail, len(r.buffer))
 	return ret
 }

pkg/appsrv/ring_test.go

@@ -19,28 +19,33 @@ import (
 )
 
 func TestRing(t *testing.T) {
-	r := NewRing(10)
-	var v int32 = 10
-	r.Push(v)
-	v = 20
-	r.Push(v)
-	v = 30
-	r.Push(v)
-	v1 := r.Pop().(int32)
-	if v1 != 10 {
-		t.Error("Fail")
-	}
-	v2 := r.Pop().(int32)
-	if v2 != 20 {
-		t.Error("Fail")
-	}
-	v3 := r.Pop().(int32)
-	if v3 != 30 {
-		t.Error("Fail")
-	}
-	v4 := r.Pop()
-	if v4 != nil {
-		t.Error("Fail")
+	var (
+		r    = NewRing(10)
+		push = func(v int32) {
+			r.Push(v)
+		}
+		pop = func(want int32) {
+			got := r.Pop().(int32)
+			if got != want {
+				t.Fatalf("got %d, want %d", got, want)
+			}
+			for i := r.header; i != r.tail; i = nextPointer(i, len(r.buffer)) {
+				if r.buffer[i] != nil {
+					t.Fatalf("head %d, tail %d, index %d not nil",
+						r.header, r.tail, i)
+				}
+			}
+		}
+	)
+	push(10)
+	push(20)
+	push(30)
+
+	pop(10)
+	pop(20)
+	pop(30)
+	if v := r.Pop(); v != nil {
+		t.Fatalf("want nil, got %#v", v)
 	}
 }
 

pkg/appsrv/workers.go

@@ -83,9 +83,6 @@ func (worker *SWorker) run() {
 			task := req.(*sWorkerTask)
 			if task.worker != nil {
 				task.worker <- worker
-				// worker channel is buffered
-				// close the worker channel
-				close(task.worker)
 			}
 			execCallback(task)
 		} else {

pkg/baremetal/cronjobs.go

@@ -189,10 +189,11 @@ func (job *SLogFetchJob) Do(ctx context.Context, now time.Time) error {
 	if err != nil {
 		return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_SYSTEM")
 	}
-	err = fetchLogs(job.baremetal, ctx, redfish.EVENT_TYPE_MANAGER)
-	if err != nil {
-		return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_MANAGER")
-	}
+	// no longer fetch management logs
+	// err = fetchLogs(job.baremetal, ctx, redfish.EVENT_TYPE_MANAGER)
+	// if err != nil {
+	// 	return errors.Wrap(err, "fetchLogs api.EVENT_TYPE_MANAGER")
+	// }
 	job.lastTime = now
 	return nil
 }

pkg/baremetal/handler/handlers.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/baremetal"
@@ -71,6 +72,7 @@ func initBaremetalsHandler(app *appsrv.Application) {
 	AddHandler(app, "POST", bmActionPrefix("reset-bmc"), bmObjMiddleware(handleBaremetalResetBMC))
 	AddHandler(app, "POST", bmActionPrefix("ipmi-probe"), bmObjMiddleware(handleBaremetalIpmiProbe))
 	AddHandler(app, "POST", bmActionPrefix("cdrom"), bmObjMiddleware(handleBaremetalCdromTask))
+	AddHandler(app, "POST", bmActionPrefix("jnlp"), bmObjMiddleware(handleBaremetalJnlpTask))
 
 	// server actions handler
 	AddHandler(app, "POST", srvActionPrefix("create"), srvClassMiddleware(handleServerCreate))
@@ -160,6 +162,17 @@ func handleBaremetalCdromTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
 	ctx.ResponseOk()
 }
 
+func handleBaremetalJnlpTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
+	jnlp, err := bm.GetConsoleJNLP(ctx)
+	if err != nil {
+		ctx.ResponseError(errors.Wrap(err, "GetConsoleJNLP"))
+		return
+	}
+	result := jsonutils.NewDict()
+	result.Add(jsonutils.NewString(jnlp), "jnlp")
+	ctx.ResponseJson(result)
+}
+
 func handleServerCreate(ctx *Context, bm *baremetal.SBaremetalInstance) {
 	err := bm.StartServerCreateTask(ctx.UserCred(), ctx.TaskId(), ctx.Data())
 	if err != nil {

pkg/baremetal/manager.go

@@ -64,6 +64,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
+	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
@@ -1232,7 +1233,12 @@ func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nic
 
 func (b *SBaremetalInstance) GetIPMIConfig() *types.SIPMIInfo {
 	conf := b.GetRawIPMIConfig()
-	if conf == nil || conf.Password == "" {
+	if conf == nil {
+		log.Debugf("GetIPMIConfig conf is nil")
+		return nil
+	}
+	if conf.Password == "" {
+		log.Debugf("GetIPMIConfig password is nil")
 		return nil
 	}
 	if conf.Username == "" {
@@ -1251,6 +1257,7 @@ func (b *SBaremetalInstance) GetIPMIConfig() *types.SIPMIInfo {
 	}
 	conf.Password = utils.Unquote(conf.Password) // XXX: remove quotes!!!
 	if conf.IpAddr == "" {
+		log.Debugf("GetIPMIConfig ipaddr s nil")
 		return nil
 	}
 	return conf
@@ -1333,6 +1340,7 @@ func (b *SBaremetalInstance) SetExistingIPMIIPAddr(ipAddr string) {
 func (b *SBaremetalInstance) GetIPMITool() *ipmitool.LanPlusIPMI {
 	conf := b.GetIPMIConfig()
 	if conf == nil {
+		log.Debugf("GetIPMIConfig is nil")
 		return nil
 	}
 	return ipmitool.NewLanPlusIPMI(conf.IpAddr, conf.Username, conf.Password)
@@ -1629,6 +1637,10 @@ func (b *SBaremetalInstance) DelayedSyncIPMIInfo(data jsonutils.JSONObject) (jso
 }
 
 func (b *SBaremetalInstance) DelayedSyncDesc(data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if data == nil {
+		session := b.manager.GetClientSession()
+		data, _ = b.manager.fetchBaremetal(session, b.GetId())
+	}
 	err := b.SaveDesc(data)
 	return nil, err
 }
@@ -1921,6 +1933,25 @@ func (b *SBaremetalInstance) fetchPowerThermalMetrics(ctx context.Context) ([]in
 	return powerMetrics, thermalMetrics, nil
 }
 
+func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error) {
+	cli := b.GetRedfishCli(ctx)
+	if cli != nil {
+		return cli.GetConsoleJNLP(ctx)
+	}
+	conf := b.GetIPMIConfig()
+	bmc := bmconsole.NewBMCConsole(conf.IpAddr, conf.Username, conf.Password, false)
+	manufacture := b.GetManufacture()
+	switch strings.ToLower(manufacture) {
+	case "hp", "hpe":
+		return bmc.GetIloConsoleJNLP(ctx)
+	case "dell", "dell inc.":
+		return bmc.GetIdracConsoleJNLP(ctx, "", "")
+	case "supermicro":
+		return bmc.GetSupermicroConsoleJNLP(ctx)
+	}
+	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
+}
+
 func (b *SBaremetalInstance) getTags() []influxdb.SKeyValue {
 	tags := []influxdb.SKeyValue{
 		{

pkg/baremetal/profiles/profiles.go

@@ -43,7 +43,7 @@ func InspurProfile() IPMIProfile {
 
 func LenovoProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{8},
+		LanChannel: []int{1, 8},
 		RootName:   "root",
 		RootId:     2,
 	}
@@ -51,7 +51,7 @@ func LenovoProfile() IPMIProfile {
 
 func HpProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{2},
+		LanChannel: []int{1, 2},
 		RootName:   "root",
 		RootId:     1,
 	}

pkg/baremetal/service/service.go

@@ -64,6 +64,7 @@ func (s *BaremetalService) StartService() {
 
 	fsdriver.Init(nil)
 	app := app_common.InitApp(&o.Options.BaseOptions, false)
+
 	handler.InitHandlers(app)
 
 	s.startAgent(app)

pkg/baremetal/tasks/base.go

@@ -306,7 +306,13 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 	if err != nil {
 		return errors.Wrapf(err, "Get power status")
 	}
+	maxTries := 10
+	count := 0
 	for status == "" || status == types.POWER_STATUS_OFF {
+		if count > maxTries {
+			break
+		}
+		log.Infof("Try power on %d times, pxe boot %v", count+1, self.PxeBoot)
 		if status == types.POWER_STATUS_OFF {
 			if self.PxeBoot {
 				err = self.Baremetal.DoPXEBoot()
@@ -314,7 +320,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				err = self.Baremetal.DoRedfishPowerOn()
 			}
 			if err != nil {
-				return errors.Wrapf(err, "Do boot power on")
+				log.Warningf("Do boot power on error: %v", err)
 			}
 		}
 		status, err = self.Baremetal.GetPowerStatus()
@@ -328,6 +334,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				return err
 			}
 		}
+		count++
 	}
 	if status != types.POWER_STATUS_ON {
 		return fmt.Errorf("Baremetal invalid restart status: %s", status)

pkg/baremetal/tasks/baseprepare.go

@@ -108,7 +108,9 @@ func (task *sBaremetalPrepareTask) prepareBaremetalInfo(cli *ssh.Client) (*barem
 	if len(raidDiskInfo) > 0 {
 		raidDrivers := []string{}
 		for _, drv := range raidDiskInfo {
-			raidDrivers = append(raidDrivers, drv.Driver)
+			if !utils.IsInStringArray(drv.Driver, raidDrivers) {
+				raidDrivers = append(raidDrivers, drv.Driver)
+			}
 		}
 		storageDriver = strings.Join(raidDrivers, ",")
 	} else {

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/app/auth.go

@@ -83,8 +83,9 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
-		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
-			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,
+		policy.EnableGlobalRbac(
+			time.Second*time.Duration(options.RbacPolicySyncPeriodSeconds),
+			time.Second*time.Duration(options.RbacPolicySyncFailedRetrySeconds),
 			options.RbacDebug,
 		)
 	}

pkg/cloudcommon/db/db_dispatcher.go

@@ -653,7 +653,7 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 			}
 		}
 		// do limit
-		if limit > 0 && total > limit {
+		if limit > 0 && total-offset > limit {
 			data = data[:limit]
 		}
 	}
@@ -1120,6 +1120,12 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 	model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId)
 	if err != nil {
 		// log.Errorf("fail to doCreateItem %s", err)
+		if CancelPendingUsagesInContext != nil {
+			err := CancelPendingUsagesInContext(ctx, userCred)
+			if err != nil {
+				log.Errorf("CancelPendingUsagesInContext fail %s", err)
+			}
+		}
 		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
 		if failErr != nil {
 			log.Errorf("manager.OnCreateFailed %s", failErr)
@@ -1308,6 +1314,19 @@ func managerPerformCheckCreateData(
 		return nil, httperrors.NewForbiddenError("not allow to perform %s", action)
 	}
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+
+		defer func() {
+			if CancelPendingUsagesInContext != nil {
+				err := CancelPendingUsagesInContext(ctx, userCred)
+				if err != nil {
+					log.Errorf("CancelPendingUsagesInContext fail %s", err)
+				}
+			}
+		}()
+	}
+
 	return ValidateCreateData(manager, ctx, userCred, ownerId, query, bodyDict)
 }
 

pkg/cloudcommon/db/modelbase.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -391,12 +390,6 @@ func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Cont
 }
 
 func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	if CancelPendingUsagesInContext != nil {
-		err := CancelPendingUsagesInContext(ctx, userCred)
-		if err != nil {
-			return errors.Wrap(err, "CancelPendingUsagesInContext")
-		}
-	}
 	return nil
 }
 

pkg/cloudcommon/db/quotas/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -435,7 +436,7 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) {
@@ -478,7 +479,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
@@ -566,3 +567,33 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 	}
 	return ret, nil
 }
+
+type tQuotaResultList []jsonutils.JSONObject
+
+func (a tQuotaResultList) Len() int           { return len(a) }
+func (a tQuotaResultList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a tQuotaResultList) Less(i, j int) bool { return usageRateOfQuota(a[i]) > usageRateOfQuota(a[j]) }
+
+func usageRateOfQuota(quota jsonutils.JSONObject) float32 {
+	maxRate := float32(0)
+	quotaMap, _ := quota.GetMap()
+	for k, v := range quotaMap {
+		usageK := fmt.Sprintf("usage.%s", k)
+		if usageV, ok := quotaMap[usageK]; ok {
+			intV, _ := v.Int()
+			if intV > 0 {
+				intUsageV, _ := usageV.Int()
+				rate := float32(intUsageV) / float32(intV)
+				if maxRate < rate {
+					maxRate = rate
+				}
+			}
+		}
+	}
+	return maxRate
+}
+
+func sortQuotaByUsage(quotaList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	sort.Sort(tQuotaResultList(quotaList))
+	return quotaList
+}

pkg/cloudcommon/db/rbac.go

@@ -70,7 +70,7 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	if !requireScope.HigherThan(scope) {
 		return nil
 	}
-	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privillege(require:%s,allow:%s)", requireScope, scope))
+	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privilege(require:%s,allow:%s)", requireScope, scope))
 }
 
 func isJointObjectRbacAllowed(item IJointModel, userCred mcclient.TokenCredential, action string, extra ...string) error {

pkg/cloudcommon/db/tenantcache.go

@@ -66,8 +66,8 @@ func RegistUserCredCacheUpdater() {
 	auth.RegisterAuthHook(onAuthCompleteUpdateCache)
 }
 
-func onAuthCompleteUpdateCache(userCred mcclient.TokenCredential) {
-	TenantCacheManager.updateTenantCache(userCred)
+func onAuthCompleteUpdateCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	TenantCacheManager.updateTenantCache(ctx, userCred)
 	UserCacheManager.updateUserCache(userCred)
 }
 
@@ -91,8 +91,8 @@ func (manager *STenantCacheManager) InitializeData() error {
 	return nil
 }
 
-func (manager *STenantCacheManager) updateTenantCache(userCred mcclient.TokenCredential) {
-	manager.Save(context.Background(), userCred.GetProjectId(), userCred.GetProjectName(),
+func (manager *STenantCacheManager) updateTenantCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	manager.Save(ctx, userCred.GetProjectId(), userCred.GetProjectName(),
 		userCred.GetProjectDomainId(), userCred.GetProjectDomain())
 }
 
@@ -348,14 +348,14 @@ func (manager *STenantCacheManager) findFirstProjectOfDomain(domainId string) (*
 	return &tenant, nil
 }
 
-func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId string) error {
+func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.Context, domainId string) error {
 	if len(domainId) == 0 {
 		log.Debugf("fetch empty domain!!!!")
 		debug.PrintStack()
 		return fmt.Errorf("Empty domainId")
 	}
 
-	s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
 	params := jsonutils.Marshal(map[string]string{"domain_id": domainId})
 	tenants, err := modules.Projects.List(s, params)
 	if err != nil {
@@ -366,7 +366,7 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 		tenantName, _ := tenant.GetString("name")
 		domainId, _ := tenant.GetString("domain_id")
 		domainName, _ := tenant.GetString("project_domain")
-		_, err = manager.Save(context.Background(), tenantId, tenantName, domainId, domainName)
+		_, err = manager.Save(ctx, tenantId, tenantName, domainId, domainName)
 		if err != nil {
 			return err
 		}
@@ -374,11 +374,11 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 	return nil
 }
 
-func (manager *STenantCacheManager) FindFirstProjectOfDomain(domainId string) (*STenant, error) {
+func (manager *STenantCacheManager) FindFirstProjectOfDomain(ctx context.Context, domainId string) (*STenant, error) {
 	tenant, err := manager.findFirstProjectOfDomain(domainId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			err = manager.fetchDomainTenantsFromKeystone(domainId)
+			err = manager.fetchDomainTenantsFromKeystone(ctx, domainId)
 			if err != nil {
 				return nil, errors.Wrap(err, "fetchDomainTenantsFromKeystone")
 			}

pkg/cloudcommon/policy/policy.go

@@ -20,6 +20,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -80,35 +81,42 @@ type SPolicyManager struct {
 	lock *sync.Mutex
 }
 
+type sPolicyData struct {
+	Type     string               `json:"type"`
+	Enabled  bool                 `json:"enabled"`
+	DomainId string               `json:"domain_id"`
+	IsPublic bool                 `json:"is_public"`
+	Policy   jsonutils.JSONObject `json:"policy"`
+}
+
 func parseJsonPolicy(obj jsonutils.JSONObject) (string, *rbacutils.SRbacPolicy, error) {
-	typeStr, err := obj.GetString("type")
+	pData := sPolicyData{}
+	err := obj.Unmarshal(&pData)
 	if err != nil {
-		return "", nil, errors.Wrap(err, "missing type")
+		return "", nil, errors.Wrap(err, "Unmarshal")
 	}
-	domainId, err := obj.GetString("domain_id")
-	if err != nil {
-		return "", nil, errors.Wrap(err, "missing domain_id")
+	if !pData.Enabled {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "not enabled")
+	}
+	if len(pData.Type) == 0 {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing type")
 	}
 
-	isPublic := jsonutils.QueryBoolean(obj, "is_public", false)
-
-	blob, err := obj.Get("policy")
-	if err != nil {
-		log.Errorf("get blob error %s", err)
-		return "", nil, errors.Wrap(err, "json.Get")
+	if pData.Policy == nil {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing policy")
 	}
 
 	policy := rbacutils.SRbacPolicy{}
-	err = policy.Decode(blob)
+	err = policy.Decode(pData.Policy)
 	if err != nil {
 		log.Errorf("policy decode error %s", err)
 		return "", nil, errors.Wrap(err, "policy.Decode")
 	}
 
-	policy.DomainId = domainId
-	policy.IsPublic = isPublic
+	policy.DomainId = pData.DomainId
+	policy.IsPublic = pData.IsPublic
 
-	return typeStr, &policy, nil
+	return pData.Type, &policy, nil
 }
 
 func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbacPolicy, error) {
@@ -121,7 +129,7 @@ func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbac
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewInt(2048), "limit")
 		params.Add(jsonutils.NewInt(int64(offset)), "offset")
-		params.Add(jsonutils.JSONTrue, "admin")
+		params.Add(jsonutils.NewString("system"), "scope")
 		params.Add(jsonutils.JSONTrue, "enabled")
 		result, err := modules.Policies.List(s, params)
 		if err != nil {
@@ -165,16 +173,38 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 			policiesMap[policy.Scope] = policies
 		}
 		manager.defaultPolicies = policiesMap
-		log.Debugf("%#v", manager.defaultPolicies)
+		// log.Debugf("%#v", manager.defaultPolicies)
 	}
 
 	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
 
-	manager.SyncOnce()
+	manager.syncByInterval()
 }
 
+func (manager *SPolicyManager) syncByInterval() {
+	syncWorkerManager.Run(manager.syncByInterval_, nil, nil)
+}
+
+func (manager *SPolicyManager) syncByInterval_() {
+	err := manager.doSync()
+	var interval time.Duration
+	if err != nil {
+		interval = manager.failedRetryInterval
+	} else {
+		interval = manager.refreshInterval
+	}
+	time.AfterFunc(interval, manager.syncByInterval)
+}
+
+var syncOnce int32
+
 func (manager *SPolicyManager) SyncOnce() {
-	syncWorkerManager.Run(manager.sync, nil, nil)
+	if atomic.CompareAndSwapInt32(&syncOnce, 0, 1) {
+		syncWorkerManager.Run(func() {
+			atomic.StoreInt32(&syncOnce, 0)
+			manager.doSync()
+		}, nil, nil)
+	}
 }
 
 func (manager *SPolicyManager) doSync() error {
@@ -202,17 +232,6 @@ func (manager *SPolicyManager) doSync() error {
 	return nil
 }
 
-func (manager *SPolicyManager) sync() {
-	err := manager.doSync()
-	var interval time.Duration
-	if err != nil {
-		interval = manager.failedRetryInterval
-	} else {
-		interval = manager.refreshInterval
-	}
-	time.AfterFunc(interval, manager.SyncOnce)
-}
-
 func queryKey(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) string {
 	queryKeys := []string{string(scope)}
 	queryKeys = append(queryKeys, userCred.GetProjectId(), userCred.GetDomainId(), userCred.GetUserId())
@@ -315,42 +334,16 @@ func (manager *SPolicyManager) findPolicyByName(scope rbacutils.TRbacScope, name
 }
 
 func getMatchedPolicyNames(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity) []string {
-	matchNames := make([]string, 0)
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchNames = matchNames[:0]
-		}
-		matchNames = append(matchNames, k)
-	}
+	_, matchNames := rbacutils.GetMatchedPolicies(policies, userCred)
 	return matchNames
 }
 
 func getMatchedPolicyRules(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity, service string, resource string, action string, extra ...string) ([]rbacutils.SRbacRule, bool) {
-	matchRules := make([]rbacutils.SRbacRule, 0)
-	findMatchPolicy := false
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchRules = matchRules[:0]
-		}
-		findMatchPolicy = true
-		rule := policies[k].GetMatchRule(service, resource, action, extra...)
-		if rule != nil {
-			matchRules = append(matchRules, *rule)
-		}
+	matchPolicies, _ := rbacutils.GetMatchedPolicies(policies, userCred)
+	if len(matchPolicies) == 0 {
+		return nil, false
 	}
-	return matchRules, findMatchPolicy
+	return matchPolicies.GetMatchRules(service, resource, action, extra...), true
 }
 
 func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
@@ -406,8 +399,15 @@ func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, use
 
 	var result rbacutils.TRbacResult
 	if len(matchRules) > 0 {
-		rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
-		result = rule.Result
+		result = rbacutils.Deny
+		for _, rule := range matchRules {
+			if rule.Result == rbacutils.Allow {
+				result = rbacutils.Allow
+				break
+			}
+		}
+		// rule := rbacutils.GetMatchRule(matchRules, service, resource, action, extra...)
+		// result = rule.Result
 	} else if findMatchPolicy {
 		// if find matched policy, but no rule matching, allow anyway
 		result = rbacutils.Allow
@@ -520,7 +520,7 @@ func (manager *SPolicyManager) IsScopeCapable(userCred mcclient.TokenCredential,
 	return false
 }
 
-func (manager *SPolicyManager) MatchedPolicies(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
+func (manager *SPolicyManager) MatchedPolicyNames(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
 	ret := make([]string, 0)
 	policies, ok := manager.policies[scope]
 	if !ok {
@@ -544,13 +544,28 @@ func (manager *SPolicyManager) AllPolicies() map[string][]string {
 }
 
 func (manager *SPolicyManager) RoleMatchPolicies(roleName string) []string {
+	ident := rbacutils.NewRbacIdentity("", "", []string{roleName})
 	ret := make([]string, 0)
 	for _, policies := range manager.policies {
 		for name, policy := range policies {
-			if policy.MatchRole(roleName) {
+			if matched, _ := policy.Match(ident); matched {
 				ret = append(ret, name)
 			}
 		}
 	}
 	return ret
 }
+
+func (manager *SPolicyManager) GetMatchedPolicySet(userCred rbacutils.IRbacIdentity) (rbacutils.TRbacScope, rbacutils.TPolicySet) {
+	for _, scope := range []rbacutils.TRbacScope{
+		rbacutils.ScopeSystem,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeProject,
+	} {
+		macthed, _ := rbacutils.GetMatchedPolicies(manager.policies[scope], userCred)
+		if len(macthed) > 0 {
+			return scope, macthed
+		}
+	}
+	return rbacutils.ScopeNone, nil
+}

pkg/cloudcommon/policy/resources.go

@@ -66,7 +66,9 @@ var (
 	meterDomainResources = []string{}
 	meterUserResources   = []string{}
 
-	k8sSystemResources = []string{}
+	k8sSystemResources = []string{
+		"repos",
+	}
 	k8sDomainResources = []string{}
 	k8sUserResources   = []string{}
 

pkg/cloudprovider/cloudprovider.go

@@ -32,33 +32,66 @@ const (
 )
 
 type SCloudaccountCredential struct {
-	ProjectName string //OpenStack
-	DomainName  string //OpenStack
-	Username    string //OpenStack Esxi ZStack
-	Password    string //OpenStack Esxi ZStack
-	AuthUrl     string //OpenStack ZStack
+	// 账号所在的项目 (openstack)
+	ProjectName string `json:"project_name"`
 
-	AccessKeyId     string //Huawei Aliyun Ucloud Aws
-	AccessKeySecret string //Huawei Aliyun Ucloud Aws
-	Environment     string //Huawei Azure Aws
+	// 账号所在的域 (openstack)
+	// default: Default
+	DomainName string `json:"domain_name"`
 
-	DirectoryId  string //Azure
-	ClientId     string //Azure
-	ClientSecret string //Azure
+	// 用户名 (openstack, zstack, esxi)
+	Username string `json:"username"`
 
-	Host string //Esxi
-	Port int    //Esxi
+	// 密码 (openstack, zstack, esxi)
+	Password string `json:"password"`
 
-	Endpoint string
+	// 认证地址 (openstack,zstack)
+	AuthUrl string `json:"auto_url"`
 
-	AppId     string //Qcloud
-	SecretId  string //Qcloud
-	SecretKey string //Qcloud
+	// 秘钥id (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
+	AccessKeyId string `json:"access_key_id"`
 
-	ClientEmail  string //Google
-	ProjectId    string //Google
-	PrivateKeyId string //Google
-	PrivateKey   string //Google
+	// 秘钥key (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
+	AccessKeySecret string `json:"access_key_secret"`
+
+	// 环境 (Azure, Aws, huawei, ctyun)
+	Environment string `json:"environment"`
+
+	// 目录ID (Azure)
+	DirectoryId string `json:"directory_id"`
+
+	// 客户端ID (Azure)
+	ClientId string `json:"client_id"`
+
+	// 客户端秘钥 (Azure)
+	ClientSecret string `json:"client_secret"`
+
+	// 主机IP (esxi)
+	Host string `json:"host"`
+
+	// 主机端口 (esxi)
+	Port int `json:"port"`
+
+	// 端点 (s3)
+	Endpoint string `json:"endpoint"`
+
+	// app id (Qcloud)
+	AppId string `json:"app_id"`
+
+	//秘钥ID (Qcloud)
+	SecretId string `json:"secret_id"`
+
+	//秘钥key (Qcloud)
+	SecretKey string `json:"secret_key"`
+
+	// Google服务账号email (gcp)
+	GCPClientEmail string `json:"gcp_client_email"`
+	// Google服务账号project id (gcp)
+	GCPProjectId string `json:"gcp_project_id"`
+	// Google服务账号秘钥id (gcp)
+	GCPPrivateKeyId string `json:"gcp_private_key_id"`
+	// Google服务账号秘钥 (gcp)
+	GCPPrivateKey string `json:"gcp_private_key"`
 }
 
 type SCloudaccount struct {
@@ -159,7 +192,7 @@ func GetProviderFactory(provider string) (ICloudProviderFactory, error) {
 	if ok {
 		return factory, nil
 	}
-	log.Errorf("Provider %s not registerd", provider)
+	log.Errorf("Provider %s not registered", provider)
 	return nil, fmt.Errorf("No such provider %s", provider)
 }
 

pkg/cloudprovider/objectstore.go

@@ -270,6 +270,7 @@ func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) {
 	if objs.IsTruncated {
 		return stats, errors.Wrap(httperrors.ErrTooLarge, "too many objects")
 	}
+	stats.ObjectCount = 0
 	for _, obj := range objs.Objects {
 		stats.SizeBytes += obj.GetSizeBytes()
 		stats.ObjectCount += 1

pkg/cloudprovider/resources.go

@@ -20,7 +20,6 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/tristate"
-	"yunion.io/x/pkg/util/secrules"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
@@ -336,10 +335,10 @@ type ICloudSecurityGroup interface {
 	ICloudResource
 
 	GetDescription() string
-	GetRules() ([]secrules.SecurityRule, error)
+	GetRules() ([]SecurityRule, error)
 	GetVpcId() string
 
-	SyncRules(rules []secrules.SecurityRule) error
+	SyncRules(common, inAdds, outAdds, inDels, outDels []SecurityRule) error
 	Delete() error
 }
 

pkg/cloudprovider/securitygroup.go

@@ -14,7 +14,15 @@
 
 package cloudprovider
 
-import "yunion.io/x/pkg/util/secrules"
+import (
+	"sort"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/secrules"
+)
+
+const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
 
 type SecurityGroupCreateInput struct {
 	Name  string
@@ -22,3 +30,268 @@ type SecurityGroupCreateInput struct {
 	VpcId string
 	Rules []secrules.SecurityRule
 }
+
+type SecurityRule struct {
+	secrules.SecurityRule
+	Name       string
+	ExternalId string
+}
+
+type TPriorityOrder int
+
+var (
+	PriorityOrderByDesc = TPriorityOrder(1)
+	PriorityOrderByAsc  = TPriorityOrder(-1)
+)
+
+func (r SecurityRule) String() string {
+	return r.SecurityRule.String()
+}
+
+type SecurityRuleSet []SecurityRule
+
+func (srs SecurityRuleSet) Len() int {
+	return len(srs)
+}
+
+func (srs SecurityRuleSet) Swap(i, j int) {
+	srs[i], srs[j] = srs[j], srs[i]
+}
+
+func (srs SecurityRuleSet) Less(i, j int) bool {
+	return srs[i].Priority < srs[j].Priority || (srs[i].Priority == srs[j].Priority && srs[i].String() < srs[j].String())
+}
+
+func (srs SecurityRuleSet) AllowList() secrules.SecurityRuleSet {
+	rules := secrules.SecurityRuleSet{}
+	for _, r := range srs {
+		rules = append(rules, r.SecurityRule)
+	}
+	return rules.AllowList()
+}
+
+func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr string, order TPriorityOrder, min, max int, onlyAllowRules bool) []SecurityRule {
+	if defaultRule.String() == localRuleStr {
+		return rules
+	}
+	defaultRule.ExternalId = DEFAULT_CLOUD_RULE_ID
+	if order == PriorityOrderByDesc {
+		defaultRule.Priority = min
+	} else {
+		defaultRule.Priority = max
+	}
+	defaultRule.Priority -= int(order)
+	if onlyAllowRules {
+		defaultRule.Priority = -1
+	}
+	return append(rules, defaultRule)
+}
+
+func SortSecurityRule(rules SecurityRuleSet, order TPriorityOrder, onlyAllowRules bool) {
+	if onlyAllowRules {
+		sort.Sort(rules)
+		return
+	}
+	if order == PriorityOrderByAsc {
+		sort.Sort(sort.Reverse(rules))
+		return
+	}
+	sort.Sort(rules)
+}
+
+func CompareRules(
+	minPriority, maxPriority int, order TPriorityOrder,
+	localRules secrules.SecurityRuleSet, remoteRules []SecurityRule,
+	defaultInRule, defaultOutRule SecurityRule,
+	onlyAllowRules bool, debug bool,
+) (common, inAdds, outAdds, inDels, outDels []SecurityRule) {
+	localInRules := secrules.SecurityRuleSet{}
+	localOutRules := secrules.SecurityRuleSet{}
+	for i := range localRules {
+		if localRules[i].Direction == secrules.DIR_IN {
+			localInRules = append(localInRules, localRules[i])
+		} else {
+			localOutRules = append(localOutRules, localRules[i])
+		}
+	}
+	inRules := SecurityRuleSet{}
+	outRules := SecurityRuleSet{}
+	for i := 0; i < len(remoteRules); i++ {
+		if remoteRules[i].Direction == secrules.DIR_IN {
+			inRules = append(inRules, remoteRules[i])
+		} else {
+			outRules = append(outRules, remoteRules[i])
+		}
+	}
+	var inCommon, outCommon = inRules, outRules
+
+	defaultLocalInRule := *secrules.MustParseSecurityRule("in:deny any")
+	defaultLocalOutRule := *secrules.MustParseSecurityRule("out:allow any")
+
+	inRules = AddDefaultRule(inRules, defaultInRule, defaultLocalInRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+	outRules = AddDefaultRule(outRules, defaultOutRule, defaultLocalOutRule.String(), order, minPriority, maxPriority, onlyAllowRules)
+
+	if defaultLocalInRule.String() != defaultInRule.String() {
+		localInRules = append(localInRules, defaultLocalInRule)
+	}
+	if defaultLocalOutRule.String() != defaultOutRule.String() {
+		localOutRules = append(localOutRules, defaultLocalOutRule)
+	}
+
+	sort.Sort(localInRules)
+	sort.Sort(localOutRules)
+
+	localInAllowList := localInRules.AllowList()
+	localOutAllowList := localOutRules.AllowList()
+	if onlyAllowRules {
+		localInRules = localInAllowList
+		localOutRules = localOutAllowList
+	}
+
+	SortSecurityRule(inRules, order, onlyAllowRules)
+	SortSecurityRule(outRules, order, onlyAllowRules)
+
+	inAllowList := inRules.AllowList()
+	outAllowList := outRules.AllowList()
+	inEquals, outEquals := inAllowList.Equals(localInAllowList), outAllowList.Equals(localOutAllowList)
+	if inEquals && outEquals {
+		return
+	}
+
+	// priority从小到大排列(从默认规则开始对比)
+	sort.Sort(sort.Reverse(localInRules))
+	sort.Sort(sort.Reverse(localOutRules))
+
+	sort.Sort(sort.Reverse(inRules))
+	sort.Sort(sort.Reverse(outRules))
+
+	startPriority := minPriority - 1
+	if order == PriorityOrderByAsc {
+		startPriority = maxPriority + 1
+	}
+
+	var addPriority = func(priority int, order TPriorityOrder, inc int, min, max int, onlyAllowRules bool) int {
+		if onlyAllowRules {
+			return 0
+		}
+		inc = inc * int(order) //+ int(order)
+		priority += inc
+		if priority < min {
+			return min
+		}
+		if priority > max {
+			return max
+		}
+		return priority
+	}
+
+	var getInitPriority = func(init, min, max int) int {
+		if init < min || init > max {
+			return (min + max) / 2
+		}
+		return init
+	}
+
+	var compare = func(localRules secrules.SecurityRuleSet, remoteRules SecurityRuleSet) (common, add, del []SecurityRule) {
+		i, j, inc, prePriority := 0, 0, 1, 0
+		for i < len(localRules) || j < len(remoteRules) {
+			if i < len(localRules) && j < len(remoteRules) {
+				ruleStr := remoteRules[j].String()
+				localRuleStr := localRules[i].String()
+				if debug {
+					log.Debugf("compare local priority(%d) %s -> remote name(%s) priority(%d) %s\n", localRules[i].Priority, localRules[i].String(), remoteRules[j].Name, remoteRules[j].Priority, remoteRules[j].String())
+				}
+				cmp := strings.Compare(ruleStr, localRuleStr)
+				if cmp == 0 {
+					prePriority = remoteRules[j].Priority
+					if remoteRules[j].ExternalId == DEFAULT_CLOUD_RULE_ID {
+						remoteRules[j].Priority = addPriority(remoteRules[j].Priority, order, 1, minPriority, maxPriority, onlyAllowRules)
+					}
+					common = append(common, remoteRules[j])
+					i++
+					j++
+				} else if cmp < 0 {
+					if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+						del = append(del, remoteRules[j])
+					}
+					j++
+				} else {
+					initPriority := getInitPriority(prePriority, minPriority, maxPriority)
+					localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+					add = append(add, SecurityRule{SecurityRule: localRules[i]})
+					i++
+					inc++
+				}
+			} else if i >= len(localRules) {
+				if remoteRules[j].ExternalId != DEFAULT_CLOUD_RULE_ID {
+					del = append(del, remoteRules[j])
+				}
+				j++
+			} else if j >= len(remoteRules) {
+				initPriority := startPriority
+				if len(remoteRules) > 0 {
+					initPriority = remoteRules[len(remoteRules)-1].Priority
+				}
+				initPriority = getInitPriority(initPriority, minPriority, maxPriority) // 若是初始添加规则，尽量以中间为节点，避免仅出现天地规则
+				localRules[i].Priority = addPriority(initPriority, order, inc, minPriority, maxPriority, onlyAllowRules)
+				add = append(add, SecurityRule{SecurityRule: localRules[i]})
+				i++
+				inc++
+			}
+		}
+		return
+	}
+
+	type rulePair struct {
+		localRules  []secrules.SecurityRule
+		remoteRules []SecurityRule
+		protocol    string
+	}
+
+	var splitRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) []rulePair {
+		rules := map[string]rulePair{}
+		for _, r := range localRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.localRules = append(pair.localRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		for _, r := range remoteRules {
+			pair, ok := rules[r.Protocol]
+			if !ok {
+				pair = rulePair{localRules: []secrules.SecurityRule{}, remoteRules: []SecurityRule{}, protocol: r.Protocol}
+			}
+			pair.remoteRules = append(pair.remoteRules, r)
+			rules[r.Protocol] = pair
+		}
+
+		ret := []rulePair{}
+		for _, r := range rules {
+			ret = append(ret, r)
+		}
+		return ret
+	}
+
+	var compareRules = func(localRules []secrules.SecurityRule, remoteRules []SecurityRule) (common, add, dels []SecurityRule) {
+		pairs := splitRules(localRules, remoteRules)
+		for _, r := range pairs {
+			_common, _add, _dels := compare(r.localRules, r.remoteRules)
+			common = append(common, _common...)
+			add = append(add, _add...)
+			dels = append(dels, _dels...)
+		}
+		return
+	}
+
+	if !inEquals {
+		inCommon, inAdds, inDels = compareRules(localInRules, inRules)
+	}
+	if !outEquals {
+		outCommon, outAdds, outDels = compareRules(localOutRules, outRules)
+	}
+	common = append(inCommon, outCommon...)
+	return
+}

pkg/compute/baremetal/diskconfig.go

@@ -549,7 +549,7 @@ func GetDiskSpecV2(storages []*BaremetalStorage) api.DiskDriverSpec {
 		if len(driverStorages) == 0 {
 			continue
 		}
-		spec[driver] = getSpec(storages)
+		spec[driver] = getSpec(driverStorages)
 	}
 	return spec
 }

pkg/compute/guestdrivers/managedvirtual.go

@@ -1009,7 +1009,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestAssociateEip(ctx context.Cont
 			return nil, fmt.Errorf("ManagedVirtualizedGuestDriver.RequestAssociateEip fail to local associate EIP %s", err)
 		}
 
-		eip.SetStatus(userCred, api.EIP_STATUS_READY, "associate")
+		eip.SetStatus(userCred, api.EIP_STATUS_READY, api.EIP_STATUS_ASSOCIATE)
 		return nil, nil
 	})
 

pkg/compute/models/buckets.go

@@ -661,10 +661,10 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	ret := jsonutils.NewDict()
 	ret.Add(retArray, "data")
+	ret.Add(jsonutils.NewString("key"), "marker_field")
+	ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	if len(nextMarker) > 0 {
 		ret.Add(jsonutils.NewString(nextMarker), "next_marker")
-		ret.Add(jsonutils.NewString("key"), "marker_field")
-		ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	}
 	return ret, nil
 }

pkg/compute/models/cloudaccounts.go

@@ -22,6 +22,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -549,13 +550,13 @@ func (self *SCloudaccount) StartSyncCloudProviderInfoTask(ctx context.Context, u
 		log.Errorf("CloudAccountSyncInfoTask newTask error %s", err)
 		return err
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
 }
 
-func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -567,7 +568,7 @@ func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) erro
 	providers := self.GetCloudproviders()
 	for i := range providers {
 		if providers[i].Enabled {
-			err := providers[i].markStartingSync(userCred)
+			err := providers[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "providers.markStartSync")
 			}
@@ -594,9 +595,9 @@ func (self *SCloudaccount) MarkEndSyncWithLock(ctx context.Context, userCred mcc
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
-	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
-		return nil
-	}
+	// if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE {
+	// 	return nil
+	// }
 
 	providers := self.GetCloudproviders()
 	for i := range providers {
@@ -717,7 +718,7 @@ func (self *SCloudaccount) importSubAccount(ctx context.Context, userCred mcclie
 				ownerId = userCred
 			} else {
 				// find default project of domain
-				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ownerId.GetProjectDomainId())
+				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ctx, ownerId.GetProjectDomainId())
 				if err != nil {
 					log.Errorf("cannot find a valid porject for domain %s", ownerId.GetProjectDomainId())
 					return nil, err
@@ -1380,6 +1381,18 @@ func (manager *SCloudaccountManager) initAllRecords() {
 	}
 }
 
+func (self *SCloudaccount) CanSync() bool {
+	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING || self.getSyncStatus2() == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING {
+		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second {
+			return true
+		} else {
+			return false
+		}
+	} else {
+		return true
+	}
+}
+
 func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 	if isStart && !options.Options.IsSlaveNode {
 		// mark all the records to be idle
@@ -1415,7 +1428,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	manager, err := account.getProviderInternal()
 	if err != nil {
 		log.Errorf("account.GetProvider failed: %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "account.getProviderInternal")
 	}
 	balance, status, err := manager.GetBalance()
 	if err != nil {
@@ -1433,7 +1446,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m
 	sysInfo, err := manager.GetSysInfo()
 	if err != nil {
 		log.Errorf("manager.GetSysInfo fail %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "manager.GetSysInfo")
 	}
 	factory := manager.GetFactory()
 	diff, err := db.Update(account, func() error {
@@ -1487,7 +1500,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 	if err != nil {
 		account.markAllProvidersDicconnected(ctx, userCred)
 		account.markAccountDiscconected(ctx, userCred)
-		return err
+		return errors.Wrap(err, "account.probeAccountStatus")
 	}
 	account.markAccountConnected(ctx, userCred)
 	providers := account.importAllSubaccounts(ctx, userCred, subaccounts)
@@ -1496,7 +1509,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc
 			_, err := providers[i].prepareCloudproviderRegions(ctx, userCred)
 			if err != nil {
 				log.Errorf("syncCloudproviderRegion fail %s", err)
-				return err
+				return errors.Wrap(err, "providers[i].prepareCloudproviderRegions")
 			}
 		}
 	}
@@ -1515,13 +1528,39 @@ func (account *SCloudaccount) markAutoSync(userCred mcclient.TokenCredential) er
 	return nil
 }
 
+var (
+	cloudaccountPendingSyncs      = map[string]struct{}{}
+	cloudaccountPendingSyncsMutex = &sync.Mutex{}
+)
+
 func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCred mcclient.TokenCredential, waitChan chan error, autoSync bool) {
+	cloudaccountPendingSyncsMutex.Lock()
+	defer cloudaccountPendingSyncsMutex.Unlock()
+	if _, ok := cloudaccountPendingSyncs[account.Id]; ok {
+		if waitChan != nil {
+			go func() {
+				// an active cloudaccount sync task is running, return with conflict error
+				log.Errorf("an active cloudaccount sync task is running, early return with conflict error")
+				waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs")
+			}()
+		}
+		return
+	}
+	cloudaccountPendingSyncs[account.Id] = struct{}{}
+
 	RunSyncCloudAccountTask(func() {
+		func() {
+			cloudaccountPendingSyncsMutex.Lock()
+			defer cloudaccountPendingSyncsMutex.Unlock()
+			delete(cloudaccountPendingSyncs, account.Id)
+		}()
+
 		log.Debugf("syncAccountStatus %s %s", account.Id, account.Name)
 		err := account.syncAccountStatus(ctx, userCred)
 		if waitChan != nil {
 			if err != nil {
 				account.markEndSync(userCred)
+				err = errors.Wrap(err, "account.syncAccountStatus")
 			}
 			waitChan <- err
 		} else {
@@ -1531,7 +1570,8 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 				account.markAutoSync(userCred)
 				providers := account.GetEnabledCloudproviders()
 				for i := range providers {
-					providers[i].syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
+					provider := &providers[i]
+					provider.syncCloudproviderRegions(ctx, userCred, syncRange, nil, autoSync)
 					syncCnt += 1
 				}
 			}

pkg/compute/models/cloudproviderregions.go

@@ -26,12 +26,14 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/nopanic"
 )
 
 type SCloudproviderregionManager struct {
@@ -225,17 +227,23 @@ func (manager *SCloudproviderregionManager) FetchByIdsOrCreate(providerId string
 	return cpr
 }
 
-func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	if !self.Enabled {
 		return fmt.Errorf("Cloudprovider(%s)region(%s) disabled", self.CloudproviderId, self.CloudregionId)
 	}
-	_, err := db.Update(self, func() error {
-		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
-		return nil
-	})
-	if err != nil {
-		log.Errorf("Failed to markStartingSync error: %v", err)
-		return err
+	regionIds := []string{}
+	if syncRange != nil {
+		regionIds, _ = syncRange.GetRegionIds()
+	}
+	if syncRange == nil || len(regionIds) == 0 || utils.IsInStringArray(self.CloudregionId, regionIds) {
+		_, err := db.Update(self, func() error {
+			self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
+			return nil
+		})
+		if err != nil {
+			log.Errorf("Failed to markStartingSync error: %v", err)
+			return err
+		}
 	}
 	return nil
 }
@@ -385,10 +393,12 @@ func (self *SCloudproviderregion) getSyncTaskKey() string {
 func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredential, syncRange SSyncRange, waitChan chan bool) {
 	self.markStartSync(userCred)
 	RunSyncCloudproviderRegionTask(self.getSyncTaskKey(), func() {
-		err := self.DoSync(context.Background(), userCred, syncRange)
-		if err != nil {
-			log.Errorf("DoSync faild %v", err)
-		}
+		nopanic.Run(func() {
+			err := self.DoSync(context.Background(), userCred, syncRange)
+			if err != nil {
+				log.Errorf("DoSync faild %v", err)
+			}
+		})
 		if waitChan != nil {
 			waitChan <- true
 		}

pkg/compute/models/cloudproviders.go

@@ -380,6 +380,44 @@ type SSyncRange struct {
 	Host   []string
 }
 
+func (sr *SSyncRange) GetRegionIds() ([]string, error) {
+	regionIds := []string{}
+	if len(sr.Host) == 0 && len(sr.Zone) == 0 && len(sr.Region) == 0 {
+		return regionIds, nil
+	}
+	hostQ := HostManager.Query().SubQuery()
+	hosts := hostQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(hostQ.Field("id"), sr.Host),
+		sqlchemy.In(hostQ.Field("name"), sr.Host),
+	)).SubQuery()
+	zoneQ := ZoneManager.Query().SubQuery()
+	zones := zoneQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(zoneQ.Field("id"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("name"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("id"), hosts.Query(hosts.Field("zone_id")).SubQuery()),
+	)).SubQuery()
+	regionQ := CloudregionManager.Query().SubQuery()
+	q := regionQ.Query(regionQ.Field("id")).Filter(sqlchemy.OR(
+		sqlchemy.In(regionQ.Field("id"), sr.Region),
+		sqlchemy.In(regionQ.Field("name"), sr.Region),
+		sqlchemy.In(regionQ.Field("id"), zones.Query(zones.Field("cloudregion_id")).SubQuery()),
+	))
+	rows, err := q.Rows()
+	if err != nil {
+		return nil, errors.Wrap(err, "q.Rows")
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var regionId string
+		err = rows.Scan(&regionId)
+		if err != nil {
+			return nil, errors.Wrap(err, "rows.Scan")
+		}
+		regionIds = append(regionIds, regionId)
+	}
+	return regionIds, nil
+}
+
 func (sr *SSyncRange) NeedSyncInfo() bool {
 	if sr.FullSync {
 		return true
@@ -535,7 +573,7 @@ func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context,
 		cloudaccount.markAutoSync(userCred)
 		cloudaccount.MarkSyncing(userCred)
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
@@ -602,7 +640,7 @@ func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred m
 	return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, DeepSync: true}, "")
 }
 
-func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
 		return nil
@@ -614,7 +652,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -623,7 +661,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	return nil
 }
 
-func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -635,7 +673,7 @@ func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) err
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -1118,9 +1156,10 @@ func (provider *SCloudprovider) GetCloudproviderRegions() []SCloudproviderregion
 func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange, wg *sync.WaitGroup, autoSync bool) {
 	provider.markSyncing(userCred)
 	cprs := provider.GetCloudproviderRegions()
+	regionIds, _ := syncRange.GetRegionIds()
 	syncCnt := 0
 	for i := range cprs {
-		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) {
+		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) && (len(regionIds) == 0 || utils.IsInStringArray(cprs[i].CloudregionId, regionIds)) {
 			syncCnt += 1
 			var waitChan chan bool = nil
 			if wg != nil {
@@ -1217,6 +1256,14 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	return nil
 }
 
+func (self *SCloudprovider) GetRegionDriver() (IRegionDriver, error) {
+	driver := GetRegionDriver(self.Provider)
+	if driver == nil {
+		return nil, fmt.Errorf("failed to found region driver for %s", self.Provider)
+	}
+	return driver, nil
+}
+
 func (self *SCloudprovider) ClearSchedDescCache() error {
 	hosts := make([]SHost, 0)
 	q := HostManager.Query().Equals("manager_id", self.Id)

pkg/compute/models/cloudregions.go

@@ -400,6 +400,9 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 	}
 
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
+		if !utils.IsInStringArray(self.Provider, api.PRIVATE_CLOUD_PROVIDERS) {
+			self.Name = cloudRegion.GetName()
+		}
 		self.Status = cloudRegion.GetStatus()
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()

pkg/compute/models/dbinstance_accounts.go

@@ -59,7 +59,7 @@ type SDBInstanceAccount struct {
 	db.SStatusStandaloneResourceBase
 	db.SExternalizedResourceBase
 
-	Secret       string `width:"256" charset:"ascii" nullable:"false" list:"domain" create:"optional"`
+	Secret       string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"optional"`
 	DBInstanceId string `width:"36" charset:"ascii" name:"dbinstance_id" nullable:"false" list:"user" create:"required" index:"true"`
 }
 
@@ -98,7 +98,7 @@ func (manager *SDBInstanceAccountManager) FetchOwnerId(ctx context.Context, data
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceAccountManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_databases.go

@@ -95,7 +95,7 @@ func (manager *SDBInstanceDatabaseManager) FetchOwnerId(ctx context.Context, dat
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceDatabaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_skus.go

@@ -59,7 +59,7 @@ type SDBInstanceSku struct {
 	SCloudregionResourceBase
 	Provider string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin"`
 
-	StorageType   string `list:"user" create:"optional"`
+	StorageType   string `width:"32" index:"true" list:"user" create:"optional"`
 	DiskSizeStep  int    `list:"user" default:"1" create:"optional"` //步长
 	MaxDiskSizeGb int    `list:"user" create:"optional"`
 	MinDiskSizeGb int    `list:"user" create:"optional"`
@@ -72,9 +72,9 @@ type SDBInstanceSku struct {
 	VcpuCount  int `nullable:"false" default:"1" list:"user" create:"optional"`
 	VmemSizeMb int `nullable:"false" list:"user" create:"required"`
 
-	Category      string `nullable:"false" list:"user" create:"optional"`
-	Engine        string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
-	EngineVersion string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	Category      string `width:"32" index:"true" nullable:"false" list:"user" create:"optional"`
+	Engine        string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	EngineVersion string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
 
 	Zone1  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`
 	Zone2  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`

pkg/compute/models/dbinstances.go

@@ -1131,9 +1131,12 @@ func (self *SDBInstance) syncRemoveCloudDBInstance(ctx context.Context, userCred
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil { // cannot delete
-		return self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.RealDelete(ctx, userCred)
 }

pkg/compute/models/disks.go

@@ -492,7 +492,9 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc
 			input.Hypervisor,
 		)
 	} else {
-		diskConfig.Backend = api.STORAGE_LOCAL
+		if len(diskConfig.Backend) == 0 {
+			diskConfig.Backend = api.STORAGE_LOCAL
+		}
 		serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor)
 		if err != nil {
 			return nil, err
@@ -901,6 +903,9 @@ func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredenti
 	}
 	addDisk := sizeMb - disk.DiskSize
 	storage := disk.GetStorage()
+	if storage == nil {
+		return httperrors.NewInternalServerError("disk has no valid storage")
+	}
 	if host := storage.GetMasterHost(); host != nil {
 		if err := host.GetHostDriver().ValidateDiskSize(storage, sizeMb>>10); err != nil {
 			return httperrors.NewInputParameterError(err.Error())
@@ -1055,6 +1060,17 @@ func (self *SDisk) ValidatePurgeCondition(ctx context.Context) error {
 }
 
 func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) error {
+	if !isPurge {
+		storage := self.GetStorage()
+		if storage == nil {
+			// storage is empty, a dirty data, allow delete
+			return nil
+		}
+		host := storage.GetMasterHost()
+		if host == nil {
+			return httperrors.NewBadRequestError("storage of disk no valid host")
+		}
+	}
 	cnt, err := self.GetGuestDiskCount()
 	if err != nil {
 		return httperrors.NewInternalServerError("GetGuestDiskCount fail %s", err)
@@ -1162,6 +1178,9 @@ func (self *SDisk) GetPathAtHost(host *SHost) string {
 
 func (self *SDisk) GetFetchUrl() string {
 	storage := self.GetStorage()
+	if storage == nil {
+		return ""
+	}
 	host := storage.GetMasterHost()
 	return fmt.Sprintf("%s/disks/%s", host.GetFetchUrl(true), self.Id)
 }
@@ -1190,7 +1209,10 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien
 	diskObj, err := db.FetchByExternalId(manager, vdisk.GetGlobalId())
 	if err != nil {
 		if err == sql.ErrNoRows {
-			vstorage, _ := vdisk.GetIStorage()
+			vstorage, err := vdisk.GetIStorage()
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to GetIStorage of vdisk %q", vdisk.GetName())
+			}
 
 			storageObj, err := db.FetchByExternalId(StorageManager, vstorage.GetGlobalId())
 			if err != nil {
@@ -1387,7 +1409,6 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	}
 	extDisk.Refresh()
 
-	storage := self.GetStorage()
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		// self.Name = extDisk.GetName()
 		self.Status = extDisk.GetStatus()
@@ -1414,8 +1435,12 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 		self.IsEmulated = extDisk.IsEmulated()
 
 		if provider.GetFactory().IsSupportPrepaidResources() && !recycle {
-			self.BillingType = extDisk.GetBillingType()
-			self.ExpiredAt = extDisk.GetExpiredAt()
+			if billintType := extDisk.GetBillingType(); len(billintType) > 0 {
+				self.BillingType = extDisk.GetBillingType()
+			}
+			if expiredAt := extDisk.GetExpiredAt(); !expiredAt.IsZero() {
+				self.ExpiredAt = extDisk.GetExpiredAt()
+			}
 		}
 
 		if createdAt := extDisk.GetCreatedAt(); !createdAt.IsZero() {
@@ -1434,6 +1459,10 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	if err != nil {
 		return errors.Wrapf(err, "Get snapshot policies of ICloudDisk %s.", extDisk.GetId())
 	}
+	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	err = SnapshotPolicyDiskManager.SyncByDisk(ctx, userCred, snapshotpolicies, syncOwnerId, self, storage)
 	if err != nil {
 		return err
@@ -1591,7 +1620,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 	// 	diskConfig.SizeMb = options.Options.DefaultDiskSize // MB
 	// else
 	if len(info.ImageId) == 0 && info.SizeMb == 0 {
-		return nil, httperrors.NewInputParameterError("Diskinfo not contains either imageID or size")
+		return nil, httperrors.NewInputParameterError("Diskinfo index %d: both imageID and size are absent", info.Index)
 	}
 	return info, nil
 }
@@ -1988,6 +2017,9 @@ func (self *SDisk) SwitchToBackup(userCred mcclient.TokenCredential) error {
 
 func (self *SDisk) ClearHostSchedCache() error {
 	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	hosts := storage.GetAllAttachingHosts()
 	if hosts == nil {
 		return fmt.Errorf("get attaching host error")
@@ -2147,7 +2179,11 @@ func (disk *SDisk) validateDiskAutoCreateSnapshot() error {
 	if len(guests) == 0 {
 		return fmt.Errorf("Disks %s not attach guest, can't create snapshot", disk.GetName())
 	}
-	if len(guests) == 1 && utils.IsInStringArray(disk.GetStorage().StorageType, api.FIEL_STORAGE) {
+	storage := disk.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
+	if len(guests) == 1 && utils.IsInStringArray(storage.StorageType, api.FIEL_STORAGE) {
 		if !utils.IsInStringArray(guests[0].Status, []string{api.VM_RUNNING, api.VM_READY}) {
 			return fmt.Errorf("Guest(%s) in status(%s) cannot do disk snapshot", guests[0].Id, guests[0].Status)
 		}
@@ -2328,6 +2364,9 @@ func (self *SDisk) GetDynamicConditionInput() *jsonutils.JSONDict {
 
 func (self *SDisk) IsNeedWaitSnapshotsDeleted() (bool, error) {
 	storage := self.GetStorage()
+	if storage == nil {
+		return false, fmt.Errorf("no valid storage")
+	}
 	if storage.StorageType == api.STORAGE_RBD {
 		scnt, err := self.GetSnapshotCount()
 		if err != nil {
@@ -2394,7 +2433,12 @@ func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCre
 	}
 	provider := self.GetCloudprovider()
 	syncOwnerId := provider.GetOwnerId()
-	region := self.GetStorage().GetRegion()
+	storage := self.GetStorage()
+	if storage == nil {
+		syncResult.Error(fmt.Errorf("no valid storage"))
+		return syncResult
+	}
+	region := storage.GetRegion()
 
 	extSnapshots, err := extDisk.GetISnapshots()
 	if err != nil {

pkg/compute/models/dnsrecords.go

@@ -36,6 +36,8 @@ type SDnsRecordManager struct {
 	db.SAdminSharableVirtualResourceBaseManager
 }
 
+var _ db.IAdminSharableVirtualModelManager = DnsRecordManager
+
 var DnsRecordManager *SDnsRecordManager
 
 func init() {
@@ -275,35 +277,38 @@ func (man *SDnsRecordManager) checkRecordValue(typ, val string) error {
 
 func (man *SDnsRecordManager) validateModelData(
 	ctx context.Context,
-	userCred mcclient.TokenCredential,
-	ownerId mcclient.IIdentityProvider,
-	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
-) (*jsonutils.JSONDict, error) {
-	records, err := man.ParseInputInfo(data)
+	isCreate bool,
+) (records []string, err error) {
+	data.Remove("records")
+	records, err = man.ParseInputInfo(data)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if len(records) == 0 {
-		return nil, httperrors.NewInputParameterError("Empty record")
+		if isCreate {
+			err = httperrors.NewInputParameterError("Empty record")
+			return
+		}
+		return
 	}
 	recType := man.getRecordsType(records)
 	name, err := data.GetString("name")
 	if err != nil {
-		return nil, err
+		return
 	}
 	err = man.checkRecordName(recType, name)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if data.Contains("ttl") {
-		jo, err := data.Get("ttl")
+		var (
+			ttl int64
+		)
+		ttl, err = data.Int("ttl")
 		if err != nil {
-			return nil, err
-		}
-		ttl, err := jo.Int()
-		if err != nil {
-			return nil, httperrors.NewInputParameterError("invalid ttl: %s", err)
+			err = httperrors.NewInputParameterError("invalid ttl: %s", err)
+			return
 		}
 		if ttl == 0 {
 			// - Create: use the database default
@@ -311,10 +316,11 @@ func (man *SDnsRecordManager) validateModelData(
 			data.Remove("ttl")
 		} else if ttl < 0 || ttl > 0x7fffffff {
 			// positive values of a signed 32 bit number.
-			return nil, httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			err = httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			return
 		}
 	}
-	return data, err
+	return records, nil
 }
 
 func (man *SDnsRecordManager) ValidateCreateData(
@@ -324,7 +330,7 @@ func (man *SDnsRecordManager) ValidateCreateData(
 	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
-	data, err := man.validateModelData(ctx, userCred, ownerId, query, data)
+	_, err := man.validateModelData(ctx, data, true)
 	if err != nil {
 		return nil, err
 	}
@@ -401,15 +407,11 @@ func (rec *SDnsRecord) GetInfo() []string {
 
 func (rec *SDnsRecord) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	data.UpdateDefault(jsonutils.Marshal(rec))
-	data, err := DnsRecordManager.validateModelData(ctx, userCred, rec.GetOwnerId(), query, data)
+	records, err := DnsRecordManager.validateModelData(ctx, data, false)
 	if err != nil {
 		return nil, err
 	}
-	{
-		records, err := DnsRecordManager.ParseInputInfo(data)
-		if err != nil {
-			return nil, err
-		}
+	if len(records) > 0 {
 		data.Set("records", jsonutils.NewString(strings.Join(records, DNS_RECORDS_SEPARATOR)))
 	}
 	return rec.SAdminSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)

pkg/compute/models/elasticcache_instances.go

@@ -116,7 +116,7 @@ func elasticcacheSubResourceFetchOwnerId(ctx context.Context, data jsonutils.JSO
 		return ec.(*SElasticcache).GetOwnerId(), nil
 	}
 
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 // elastic cache 子资源获取owner query
@@ -329,9 +329,12 @@ func (self *SElasticcache) syncRemoveCloudElasticcache(ctx context.Context, user
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
+	self.DeletePreventionOff(self, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
-		return self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")
+		return errors.Wrap(err, "ValidateDeleteCondition")
 	}
 	return self.Delete(ctx, userCred)
 }

pkg/compute/models/external_projects.go

@@ -185,8 +185,10 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
-		self.ProjectId = provider.ProjectId
-		self.DomainId = provider.DomainId
+		if self.DomainId != provider.DomainId {
+			self.ProjectId = provider.ProjectId
+			self.DomainId = provider.DomainId
+		}
 		return nil
 	})
 	if err != nil {

pkg/compute/models/guest_actions.go

@@ -3915,6 +3915,20 @@ func (self *SGuest) PerformSyncFixNics(ctx context.Context,
 		return nil, httperrors.NewInputParameterError("missing field ip, list of ip")
 	}
 	iplist := iplistArray.(*jsonutils.JSONArray).GetStringArray()
+	errs := make([]error, 0)
+	for i := range vnics {
+		ip := vnics[i].GetIP()
+		if len(ip) == 0 {
+			continue
+		}
+		_, err := host.getNetworkOfIPOnHost(ip)
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, ip))
+		}
+	}
+	if len(errs) > 0 {
+		return nil, httperrors.NewInvalidStatusError(errors.NewAggregate(errs).Error())
+	}
 	result := self.SyncVMNics(ctx, userCred, host, vnics, iplist)
 	if result.IsError() {
 		return nil, httperrors.NewInternalServerError(result.Result())
@@ -4385,6 +4399,9 @@ func (self *SGuest) PerformSnapshotAndClone(
 		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
+	} else {
+		cancelRegionUsage := &SRegionQuota{Snapshot: snapshotUsage.Snapshot}
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, cancelRegionUsage, true)
 	}
 
 	err = self.StartInstanceSnapshotAndCloneTask(

pkg/compute/models/guestdisks.go

@@ -183,15 +183,11 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 			desc.Add(jsonutils.NewString(storagecacheimg.Path), "image_path")
 		}
 	}
-	storage := disk.GetStorage()
-	// XXX ???
 	if host.HostType == api.HOST_TYPE_HYPERVISOR {
 		desc.Add(jsonutils.NewString(disk.StorageId), "storage_id")
 		localpath := disk.GetPathAtHost(host)
 		if len(localpath) == 0 {
 			desc.Add(jsonutils.JSONTrue, "migrating")
-			target := host.GetLeastUsedStorage(storage.StorageType)
-			desc.Add(jsonutils.NewString(target.Id), "target_storage_id")
 			disk.SetStatus(nil, api.DISK_START_MIGRATE, "migration")
 		} else {
 			desc.Add(jsonutils.NewString(localpath), "path")

pkg/compute/models/guestnetworks.go

@@ -16,11 +16,9 @@ package models
 
 import (
 	"context"
-	"crypto/md5"
-	"crypto/rand"
 	"database/sql"
 	"fmt"
-	"io"
+	"math/rand"
 	"regexp"
 	"time"
 
@@ -37,6 +35,7 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	randutil "yunion.io/x/onecloud/pkg/util/rand"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -226,17 +225,6 @@ func (manager *SGuestnetworkManager) newGuestNetwork(ctx context.Context, userCr
 	return &gn, nil
 }
 
-func (self *SGuestnetwork) getVirtualRand(width int, randomized bool) string {
-	hash := md5.New()
-	io.WriteString(hash, self.GuestId)
-	io.WriteString(hash, self.NetworkId)
-	if randomized {
-		io.WriteString(hash, fmt.Sprintf("%d", time.Now().Unix()))
-	}
-	hex := fmt.Sprintf("%x", hash.Sum(nil))
-	return hex[:width]
-}
-
 func (self *SGuestnetwork) generateIfname(network *SNetwork, virtual bool, randomized bool) string {
 	// It may happen that external networks when synced can miss ifname hint
 	network.ensureIfnameHint()
@@ -247,8 +235,7 @@ func (self *SGuestnetwork) generateIfname(network *SNetwork, virtual bool, rando
 		nName = nName[:(MAX_IFNAME_SIZE - 4)]
 	}
 	if virtual {
-		rand := self.getVirtualRand(3, randomized)
-		return fmt.Sprintf("%s-%s", nName, rand)
+		return fmt.Sprintf("%s-%s", nName, randutil.String(3))
 	} else {
 		ip, _ := netutils.NewIPV4Addr(self.IpAddr)
 		cliaddr := ip.CliAddr(network.GuestIpMask)

pkg/compute/models/guests.go

@@ -1129,6 +1129,9 @@ func (manager *SGuestManager) validateCreateData(
 				rootDiskConfig.SizeMb = sysMinDiskMB
 			}
 		}
+		if len(rootDiskConfig.Driver) == 0 {
+			rootDiskConfig.Driver = osProf.DiskDriver
+		}
 		log.Debugf("ROOT DISK: %#v", rootDiskConfig)
 		input.Disks[0] = rootDiskConfig
 		//data.Set("disk.0", jsonutils.Marshal(rootDiskConfig))
@@ -1518,10 +1521,13 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 		guest.setUserData(ctx, userCred, userData)
 	}
 	secgroups, _ := jsonutils.GetStringArray(data, "secgroups")
-	for _, secgroup := range secgroups {
-		gs := SGuestsecgroup{SecgroupId: secgroup}
-		gs.GuestId = guest.Id
-		GuestsecgroupManager.TableSpec().Insert(&gs)
+	for _, secgroupId := range secgroups {
+		if secgroupId != guest.SecgrpId {
+			gs := SGuestsecgroup{}
+			gs.SecgroupId = secgroupId
+			gs.GuestId = guest.Id
+			GuestsecgroupManager.TableSpec().Insert(&gs)
+		}
 	}
 }
 
@@ -3205,14 +3211,19 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 		return nil, err
 	}
 
-	cancelUsage := SQuota{}
-	cancelUsage.Storage = disk.DiskSize
-	keys, err := self.GetQuotaKeys()
-	if err != nil {
-		return nil, err
+	if pendingUsage != nil {
+		cancelUsage := SQuota{}
+		cancelUsage.Storage = disk.DiskSize
+		keys, err := self.GetQuotaKeys()
+		if err != nil {
+			return nil, err
+		}
+		cancelUsage.SetKeys(keys)
+		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
+		if err != nil {
+			return nil, err
+		}
 	}
-	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 
 	return disk, nil
 }
@@ -4701,7 +4712,9 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 		for idx, disk := range genInput.Disks {
 			tmpD := disk
 			if idx < len(userInput.Disks) {
-				tmpD.Schedtags = userInput.Disks[idx].Schedtags
+				inputDisk := userInput.Disks[idx]
+				tmpD.Schedtags = inputDisk.Schedtags
+				tmpD.Storage = inputDisk.Storage
 			}
 			disks = append(disks, tmpD)
 		}
@@ -4711,7 +4724,9 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 	for idx, net := range genInput.Networks {
 		tmpN := net
 		if idx < len(userInput.Networks) {
-			tmpN.Schedtags = userInput.Disks[idx].Schedtags
+			inputNet := userInput.Networks[idx]
+			tmpN.Schedtags = inputNet.Schedtags
+			tmpN.Network = inputNet.Network
 		}
 		nets = append(nets, tmpN)
 	}

pkg/compute/models/hosts.go

@@ -53,7 +53,6 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/logclient"
-	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 )
 
 type SHostManager struct {
@@ -3631,10 +3630,16 @@ func (self *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCrede
 			}
 			net, err = wire.GetCandidatePrivateNetwork(userCred, false, netTypes)
 			if err != nil {
-				return fmt.Errorf("fail to find network %s", err)
+				return fmt.Errorf("fail to find private network %s", err)
 			}
 			if net == nil {
-				return fmt.Errorf("No network found")
+				net, err = wire.GetCandidatePublicNetwork(false, netTypes)
+				if err != nil {
+					return fmt.Errorf("fail to find public network %s", err)
+				}
+				if net == nil {
+					return fmt.Errorf("No network found")
+				}
 			}
 		}
 	} else if net.WireId != wire.Id {
@@ -4585,39 +4590,13 @@ func (self *SHost) AllowGetDetailsJnlp(ctx context.Context, userCred mcclient.To
 }
 
 func (self *SHost) GetDetailsJnlp(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	ipmi, err := self.GetIpmiInfo()
+	url := fmt.Sprintf("/baremetals/%s/jnlp", self.Id)
+	header := mcclient.GetTokenHeaders(userCred)
+	resp, err := self.BaremetalSyncRequest(ctx, "POST", url, header, nil)
 	if err != nil {
-		return nil, httperrors.NewInvalidStatusError("no valid ipmi_info")
+		return nil, errors.Wrap(err, "BaremetalSyncRequest")
 	}
-	if !ipmi.Verified {
-		return nil, httperrors.NewInvalidStatusError("no veried ipmi_info")
-	}
-	if self.SysInfo == nil {
-		return nil, httperrors.NewInvalidStatusError("no valid sys_info")
-	}
-	ipmiPass, err := utils.DescryptAESBase64(self.Id, ipmi.Password)
-	if err != nil {
-		return nil, httperrors.NewInternalServerError("decrypt ipmi password fail: %s", err)
-	}
-	bmc := bmconsole.NewBMCConsole(ipmi.IpAddr, ipmi.Username, ipmiPass, false)
-	manufacture, _ := self.SysInfo.GetString("manufacture")
-	var jnlp string
-	switch strings.ToLower(manufacture) {
-	case "hp", "hpe":
-		jnlp, err = bmc.GetIloConsoleJNLP(ctx)
-	case "dell", "dell inc.":
-		sku, _ := self.SysInfo.GetString("sku")
-		model, _ := self.SysInfo.GetString("model")
-		jnlp, err = bmc.GetIdracConsoleJNLP(ctx, sku, model)
-	default:
-		return nil, httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
-	}
-	if err != nil {
-		return nil, httperrors.NewGeneralError(err)
-	}
-	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.NewString(jnlp), "jnlp")
-	return ret, nil
+	return resp, nil
 }
 
 func (self *SHost) AllowPerformInsertIso(ctx context.Context,

pkg/compute/models/loadbalanceragents.go

@@ -147,6 +147,9 @@ func (p *SLoadbalancerAgentParamsVrrp) validatePeer(pp *SLoadbalancerAgentParams
 	if p.VirtualRouterId != pp.VirtualRouterId {
 		return fmt.Errorf("vrrp virtual_router_id of peer lbagents must be the same: %d != %d", p.VirtualRouterId, pp.VirtualRouterId)
 	}
+	if p.AdvertInt != pp.AdvertInt {
+		return fmt.Errorf("vrrp advert_int of peer lbagents must be the same: %d != %d", p.AdvertInt, pp.AdvertInt)
+	}
 	if p.Preempt != pp.Preempt {
 		return fmt.Errorf("vrrp preempt property of peer lbagents must be the same: %v != %v", p.Preempt, pp.Preempt)
 	}
@@ -161,6 +164,9 @@ func (p *SLoadbalancerAgentParamsVrrp) needsUpdatePeer(pp *SLoadbalancerAgentPar
 	if p.VirtualRouterId != pp.VirtualRouterId {
 		return true
 	}
+	if p.AdvertInt != pp.AdvertInt {
+		return true
+	}
 	if p.Preempt != pp.Preempt {
 		return true
 	}
@@ -172,6 +178,7 @@ func (p *SLoadbalancerAgentParamsVrrp) needsUpdatePeer(pp *SLoadbalancerAgentPar
 
 func (p *SLoadbalancerAgentParamsVrrp) updateBy(pp *SLoadbalancerAgentParamsVrrp) {
 	p.VirtualRouterId = pp.VirtualRouterId
+	p.AdvertInt = pp.AdvertInt
 	p.Preempt = pp.Preempt
 	p.Pass = pp.Pass
 }
@@ -431,9 +438,11 @@ func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx contex
 	}
 	agentsData := jsonutils.Marshal(&agents).(*jsonutils.JSONArray)
 	for fieldName, man := range men {
-		keyPlural := man.KeywordPlural()
-		now := time.Now()
-		minT := now
+		var (
+			keyPlural = man.KeywordPlural()
+			now       = time.Now()
+			minT      = now
+		)
 		if len(agents) > 0 {
 			// find min updated_at seen by these active agents
 			for i := 0; i < agentsData.Length(); i++ {
@@ -462,17 +471,13 @@ func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx contex
 				continue
 			}
 			defer rows.Close()
-			m, err := db.NewModelObject(man)
-			if err != nil {
-				log.Errorf("%s: new model object failed: %s", keyPlural, err)
-				continue
-			}
-			mInitValue := reflect.Indirect(reflect.ValueOf(m))
-			m, _ = db.NewModelObject(man)
 			for rows.Next() {
-				reflect.Indirect(reflect.ValueOf(m)).Set(mInitValue)
-				err := q.Row2Struct(rows, m)
+				m, err := db.NewModelObject(man)
 				if err != nil {
+					log.Errorf("%s: new model object failed: %s", keyPlural, err)
+					continue
+				}
+				if err := q.Row2Struct(rows, m); err != nil {
 					log.Errorf("%s: Row2Struct: %s", keyPlural, err)
 					continue
 				}

pkg/compute/models/loadbalanceragents_deploy.go

@@ -159,7 +159,7 @@ func (lbagent *SLoadbalancerAgent) deploy(ctx context.Context, userCred mcclient
 				return nil, errors.WithMessagef(err, "glob error %s", pattern)
 			}
 			if len(matches) == 0 {
-				return nil, errors.WithMessagef(err, "glob nomatch %s", pattern)
+				return nil, errors.Errorf("no match for %q", pattern)
 			}
 			path := matches[len(matches)-1]
 			name := filepath.Base(path)
@@ -223,12 +223,9 @@ func (lbagent *SLoadbalancerAgent) undeploy(ctx context.Context, userCred mcclie
 		},
 		Modules: []ansible.Module{
 			{
-				Name: "systemd",
+				Name: "shell",
 				Args: []string{
-					"name=yunion-lbagent",
-					"enabled=no",
-					"state=stopped",
-					"daemon_reload=yes",
+					"systemctl disable --now yunion-lbagent; true",
 				},
 			},
 			{

pkg/compute/models/loadbalancerawscachedlbbg.go

@@ -234,14 +234,8 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 	}
 	for i := 0; i < len(commondb); i++ {
 		var elb *SLoadbalancer
-		elbIds := commonext[i].GetLoadbalancerId()
-		if err != nil {
-			syncResult.UpdateError(err)
-			continue
-		}
-
 		elbId := commonext[i].GetLoadbalancerId()
-		if len(elbIds) > 0 {
+		if len(elbId) > 0 {
 			ielb, err := db.FetchByExternalId(LoadbalancerManager, elbId)
 			if err == nil {
 				elb = ielb.(*SLoadbalancer)
@@ -249,10 +243,8 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 		}
 
 		if elb == nil {
-			elb = &SLoadbalancer{}
-			elb.Id = ""
-			elb.CloudregionId = region.GetId()
-			elb.ManagerId = provider.GetId()
+			log.Debugf("Aws.SyncLoadbalancerBackendgroups skiped external backendgroup %s", elbId)
+			continue
 		}
 
 		err = commondb[i].SyncWithCloudLoadbalancerBackendgroup(ctx, userCred, elb, commonext[i], provider.GetOwnerId())
@@ -269,23 +261,18 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 	for i := 0; i < len(added); i++ {
 		var elb *SLoadbalancer
 		elbId := added[i].GetLoadbalancerId()
-		if err != nil {
-			syncResult.AddError(err)
-			continue
-		}
-
 		if len(elbId) > 0 {
 			elb, err = LoadbalancerManager.FetchByExternalId(provider.GetId(), elbId)
 			if err != nil {
 				log.Debugf("awsCachedLbbgManager.SyncLoadbalancerBackendgroups %s", err)
+				syncResult.AddError(err)
+				continue
 			}
 		}
 
 		if elb == nil {
-			elb = &SLoadbalancer{}
-			elb.Id = ""
-			elb.CloudregionId = region.GetId()
-			elb.ManagerId = provider.GetId()
+			log.Debugf("Aws.SyncLoadbalancerBackendgroups skiped external backendgroup %s", elbId)
+			continue
 		}
 
 		new, err := man.newFromCloudLoadbalancerBackendgroup(ctx, userCred, elb, added[i], syncOwnerId)

pkg/compute/models/loadbalancerbackendgroups.go

@@ -101,6 +101,15 @@ func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context,
 	return q, nil
 }
 
+func (man *SLoadbalancerBackendGroupManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", nil)
+	err := lbV.Validate(data.(*jsonutils.JSONDict))
+	if err == nil {
+		return lbV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerId)
 	err := lbV.Validate(data)
@@ -929,12 +938,29 @@ func (man *SLoadbalancerBackendGroupManager) initBackendGroupType() error {
 }
 
 func (man *SLoadbalancerBackendGroupManager) InitializeData() error {
-	if err := man.initBackendGroupType(); err != nil {
-		return err
+	q := man.Query().IsNullOrEmpty("loadbalancer_id")
+	lbbgs := make([]SLoadbalancerBackendGroup, 0)
+	err := db.FetchModelObjects(man, q, &lbbgs)
+	if err != nil {
+		return errors.Wrap(err, "SLoadbalancerBackendGroupManager.InitializeData")
 	}
-	return man.initBackendGroupRegion()
+
+	for i := range lbbgs {
+		lbbg := lbbgs[i]
+		_, err = db.UpdateWithLock(context.Background(), &lbbg, func() error {
+			lbbg.MarkDelete()
+			return nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "SLoadbalancerBackendGroupManager.InitializeData.MarkDelete")
+		}
+	}
+
+	log.Debugf("SLoadbalancerBackendGroupManager.InitializeData removed %d invalid loadbalancer backendgroup.", len(lbbgs))
+	return nil
 }
 
+/*
 func (manager *SLoadbalancerBackendGroupManager) initBackendGroupRegion() error {
 	groups := []SLoadbalancerBackendGroup{}
 	q := manager.Query()
@@ -956,7 +982,7 @@ func (manager *SLoadbalancerBackendGroupManager) initBackendGroupRegion() error
 		}
 	}
 	return nil
-}
+}*/
 
 func (manager *SLoadbalancerBackendGroupManager) GetResourceCount() ([]db.SProjectResourceCount, error) {
 	virts := manager.Query().IsFalse("pending_deleted")

pkg/compute/models/loadbalancerbackends.go

@@ -145,6 +145,14 @@ func (man *SLoadbalancerBackendManager) ValidateBackendVpc(lb *SLoadbalancer, gu
 	return nil
 }
 
+func (man *SLoadbalancerBackendManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", nil)
+	if err := backendGroupV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return backendGroupV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerId)
 	if err := backendGroupV.Validate(data); err != nil {

pkg/compute/models/loadbalancerlistenerrules.go

@@ -397,6 +397,14 @@ func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context,
 	return q, nil
 }
 
+func (man *SLoadbalancerListenerRuleManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", nil)
+	if err := listenerV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return listenerV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	input := apis.VirtualResourceCreateInput{}
 	err := data.Unmarshal(&input)

pkg/compute/models/loadbalancerlisteners.go

@@ -224,6 +224,14 @@ func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q *
 	return q, nil
 }
 
+func (man *SLoadbalancerListenerManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", nil)
+	if err := lbV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return lbV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerId)
 	if err := lbV.Validate(data); err != nil {
@@ -345,7 +353,7 @@ func (lblis *SLoadbalancerListener) StartLoadBalancerListenerSyncstatusTask(ctx
 func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	ownerId := lblis.GetOwnerId()
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerId)
-	backendGroupV.AllowEmpty(true).Optional(true)
+	backendGroupV.AllowEmpty(true).Default(lblis.BackendGroupId)
 	if err := backendGroupV.Validate(data); err != nil {
 		return nil, err
 	}

pkg/compute/models/loadbalancers.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -318,7 +319,8 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 	if lb.ChargeType == api.LB_CHARGE_TYPE_BY_BANDWIDTH {
 		params.EgressMbps = lb.EgressMbps
 	}
-	if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || lb.GetProviderName() == api.CLOUD_PROVIDER_HUAWEI || lb.GetProviderName() == api.CLOUD_PROVIDER_AWS {
+
+	if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_AWS, api.CLOUD_PROVIDER_QCLOUD}) {
 		vpc := lb.GetVpc()
 		if vpc == nil {
 			return nil, fmt.Errorf("failed to find vpc for lb %s", lb.Name)
@@ -328,6 +330,9 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou
 			return nil, err
 		}
 		params.VpcID = iVpc.GetId()
+	}
+
+	if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_AWS}) {
 		networks, err := lb.GetNetworks()
 		if err != nil {
 			return nil, fmt.Errorf("failed to find network for lb %s: %s", lb.Name, err)
@@ -775,11 +780,18 @@ func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred
 		lb.LoadbalancerSpec = extLb.GetLoadbalancerSpec()
 		lb.EgressMbps = extLb.GetEgressMbps()
 		lb.ChargeType = extLb.GetChargeType()
-
+		lbNetworkIds := getExtLbNetworkIds(extLb)
+		lb.NetworkId = strings.Join(lbNetworkIds, ",")
 		if extLb.GetMetadata() != nil {
 			lb.LBInfo = extLb.GetMetadata()
 		}
 
+		if vpcId := extLb.GetVpcId(); len(vpcId) > 0 {
+			if vpc, err := db.FetchByExternalId(VpcManager, vpcId); err == nil && vpc != nil {
+				lb.VpcId = vpc.GetId()
+			}
+		}
+
 		return nil
 	})
 

pkg/compute/models/networks.go

@@ -50,10 +50,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
-var (
-	ALL_NETWORK_TYPES = api.ALL_NETWORK_TYPES
-)
-
 type SNetworkManager struct {
 	db.SSharableVirtualResourceBaseManager
 }
@@ -745,10 +741,11 @@ func (manager *SNetworkManager) GetOnPremiseNetworkOfIP(ipAddr string, serverTyp
 	}
 	q := manager.Query()
 	wires := WireManager.Query().SubQuery()
-	vpcs := VpcManager.Query().SubQuery()
+	// vpcs := VpcManager.Query().SubQuery()
 	q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id")))
-	q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
-	q = q.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
+	// q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
+	// q = q.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
+	q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), api.DEFAULT_VPC_ID))
 	if len(serverType) > 0 {
 		q = q.Filter(sqlchemy.Equals(q.Field("server_type"), serverType))
 	}
@@ -1420,7 +1417,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 
 	if len(input.ServerType) == 0 {
 		input.ServerType = api.NETWORK_TYPE_GUEST
-	} else if !utils.IsInStringArray(input.ServerType, ALL_NETWORK_TYPES) {
+	} else if !utils.IsInStringArray(input.ServerType, api.ALL_NETWORK_TYPES) {
 		return input, httperrors.NewInputParameterError("Invalid server_type: %s", input.ServerType)
 	}
 

pkg/compute/models/regiondrivers.go

@@ -116,6 +116,12 @@ type IRegionDriver interface {
 
 	RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, task taskman.ITask) error
 	RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup) (string, error)
+	GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder // Desc(priority值越大,优先级越高) Asc(priority值越小,优先级越高)
+	GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule
+	GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule
+	GetSecurityGroupRuleMaxPriority() int
+	GetSecurityGroupRuleMinPriority() int
+	IsOnlySupportAllowRules() bool
 	IsSupportClassicSecurityGroup() bool
 	IsSecurityGroupBelongVpc() bool
 	IsVpcBelongGlobalVpc() bool

pkg/compute/models/secgroupcache.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SSecurityGroupCacheManager struct {
@@ -104,6 +105,25 @@ func (self *SSecurityGroupCache) GetIRegion() (cloudprovider.ICloudRegion, error
 	return nil, fmt.Errorf("failed to find iregion for secgroupcache %s vpc: %s externalId: %s", self.Id, self.VpcId, self.ExternalId)
 }
 
+func (manager *SSecurityGroupCacheManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
+	if userCred != nil {
+		sq := SecurityGroupManager.Query("id")
+		switch scope {
+		case rbacutils.ScopeProject:
+			if len(userCred.GetProjectId()) > 0 {
+				sq = sq.Equals("tenant_id", userCred.GetProjectId())
+				return q.In("secgroup_id", sq)
+			}
+		case rbacutils.ScopeDomain:
+			if len(userCred.GetProjectDomainId()) > 0 {
+				sq = sq.Equals("domain_id", userCred.GetProjectDomainId())
+				return q.In("secgroup_id", sq)
+			}
+		}
+	}
+	return q
+}
+
 func (self *SSecurityGroupCache) GetVpc() (*SVpc, error) {
 	vpc, err := VpcManager.FetchById(self.VpcId)
 	if err != nil {

pkg/compute/models/secgrouprules.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -113,7 +114,7 @@ func (manager *SSecurityGroupRuleManager) FetchOwnerId(ctx context.Context, data
 		}
 		return secgroup.(*SSecurityGroup).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SSecurityGroupRuleManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
@@ -335,13 +336,13 @@ func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSec
 	return rules, nil
 }
 
-func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules secrules.SecurityRuleSet) compare.SyncResult {
+func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules cloudprovider.SecurityRuleSet) compare.SyncResult {
 	syncResult := compare.SyncResult{}
-	priority, prePriority := 100, 0
+	priority, prePriority := 10, 0
 	for i := 0; i < len(rules); i++ {
 		// 这里避免了Rule规则优先级在 1-100之外的问题,ext.GetRules()不需要进行优先级转换
-		if prePriority != 0 && rules[i].Priority != prePriority && priority > 1 {
-			priority--
+		if prePriority != 0 && rules[i].Priority != prePriority && priority < 100 {
+			priority++
 		}
 		prePriority = rules[i].Priority
 		rules[i].Priority = priority
@@ -355,7 +356,7 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre
 	return syncResult
 }
 
-func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, rule secrules.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) {
+func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, rule cloudprovider.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 

pkg/compute/models/secgroups.go

@@ -17,7 +17,6 @@ package models
 import (
 	"context"
 	"database/sql"
-	"sort"
 	"strings"
 	"time"
 
@@ -645,44 +644,46 @@ func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, err
 }
 
 func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) {
+	regionDriver, err := provider.GetRegionDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "provider.GetRegionDriver")
+	}
+
 	rules, err := extSec.GetRules()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "extSec.GetRules")
 	}
-	inRules := secrules.SecurityRuleSet{}
-	outRules := secrules.SecurityRuleSet{}
-	for i := 0; i < len(rules); i++ {
+
+	inRules := []cloudprovider.SecurityRule{}
+	outRules := []cloudprovider.SecurityRule{}
+	for i := range rules {
 		if rules[i].Direction == secrules.DIR_IN {
 			inRules = append(inRules, rules[i])
 		} else {
 			outRules = append(outRules, rules[i])
 		}
 	}
-	sort.Sort(inRules)
-	sort.Sort(outRules)
-	inAllowList := inRules.AllowList()
-	outAllowList := outRules.AllowList()
+
+	maxPriority := regionDriver.GetSecurityGroupRuleMaxPriority()
+	minPriority := regionDriver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := regionDriver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := regionDriver.GetDefaultSecurityGroupOutRule()
+	order := regionDriver.GetSecurityGroupRuleOrder()
+	onlyAllowRules := regionDriver.IsOnlySupportAllowRules()
 
 	// 查询所有共享或与provider在同一项目的安全组，比对寻找一个与云上安全组规则相同的安全组
 	secgroups := []SSecurityGroup{}
-	q := manager.Query()
-	q = q.Filter(
-		sqlchemy.OR(
-			sqlchemy.Equals(q.Field("tenant_id"), provider.ProjectId),
-			sqlchemy.AND(
-				sqlchemy.IsTrue(q.Field("is_public")),
-				sqlchemy.Equals(q.Field("public_scope"), rbacutils.ScopeSystem),
-			),
-		),
-	)
-	if err := db.FetchModelObjects(manager, q, &secgroups); err != nil {
-		log.Errorf("failed to fetch secgroups %v", err)
+	q := manager.Query().Equals("domain_id", provider.DomainId)
+	err = db.FetchModelObjects(manager, q, &secgroups)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
 	}
-	for _, secgroup := range secgroups {
-		_inAllowList := secgroup.GetInAllowList()
-		_outAllowList := secgroup.GetOutAllowList()
-		if outAllowList.Equals(_outAllowList) && inAllowList.Equals(_inAllowList) {
-			return &secgroup, nil
+	for i := range secgroups {
+		localRules := secrules.SecurityRuleSet(secgroups[i].GetSecRules(""))
+		_, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
+		if len(inAdds) == 0 && len(outAdds) == 0 && len(inDels) == 0 && len(outDels) == 0 {
+			return &secgroups[i], nil
 		}
 	}
 
@@ -695,6 +696,7 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 	if err != nil {
 		return nil, err
 	}
+
 	secgroup.Name = newName
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
@@ -705,6 +707,11 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 	}
 
 	//这里必须先同步下规则,不然下次对比此安全组规则为空
+	inRules = cloudprovider.AddDefaultRule(inRules, defaultInRule, "in:deny any", order, minPriority, maxPriority, onlyAllowRules)
+	cloudprovider.SortSecurityRule(inRules, order, onlyAllowRules)
+	outRules = cloudprovider.AddDefaultRule(outRules, defaultOutRule, "out:allow any", order, minPriority, maxPriority, onlyAllowRules)
+	cloudprovider.SortSecurityRule(outRules, order, onlyAllowRules)
+
 	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, inRules)
 	SecurityGroupRuleManager.SyncRules(ctx, userCred, &secgroup, outRules)
 
@@ -719,18 +726,21 @@ func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mc
 	} else {
 		needSync := false
 
-		lockman.LockObject(ctx, secgrp)
-		defer lockman.ReleaseObject(ctx, secgrp)
+		func() {
+			lockman.LockObject(ctx, secgrp)
+			defer lockman.ReleaseObject(ctx, secgrp)
 
-		if secgrp.IsDirty {
-			if _, err := db.Update(secgrp, func() error {
-				secgrp.IsDirty = false
-				return nil
-			}); err != nil {
-				log.Errorf("Update Security Group error: %s", err.Error())
+			if secgrp.IsDirty {
+				if _, err := db.Update(secgrp, func() error {
+					secgrp.IsDirty = false
+					return nil
+				}); err != nil {
+					log.Errorf("Update Security Group error: %s", err.Error())
+				}
+				needSync = true
 			}
-			needSync = true
-		}
+		}()
+
 		if needSync {
 			for _, guest := range secgrp.GetGuests() {
 				guest.StartSyncTask(ctx, userCred, true, "")
@@ -747,7 +757,7 @@ func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenC
 		log.Errorf("Update Security Group error: %s", err.Error())
 	}
 	time.AfterFunc(10*time.Second, func() {
-		SecurityGroupManager.DelaySync(ctx, userCred, self.Id)
+		SecurityGroupManager.DelaySync(context.Background(), userCred, self.Id)
 	})
 }
 

pkg/compute/models/snapshots.go

@@ -384,9 +384,8 @@ func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string)
 
 func (self *SSnapshotManager) GetDiskSnapshots(diskId string) []SSnapshot {
 	dest := make([]SSnapshot, 0)
-	q := self.Query().SubQuery()
-	sq := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId)))
-	err := db.FetchModelObjects(self, sq, &dest)
+	q := self.Query().Equals("disk_id", diskId).Asc("created_at")
+	err := db.FetchModelObjects(self, q, &dest)
 	if err != nil {
 		log.Errorf("GetDiskSnapshots error: %s", err)
 		return nil

pkg/compute/models/wires.go

@@ -108,11 +108,10 @@ func (manager *SWireManager) ValidateCreateData(ctx context.Context, userCred mc
 	}
 
 	vpcStr := jsonutils.GetAnyString(data, []string{"vpc", "vpc_id"})
-	if len(vpcStr) == 0 {
-		return nil, httperrors.NewMissingParameterError("vpc_id")
+	if vpcStr == "" {
+		vpcStr = api.DEFAULT_VPC_ID
 	}
-
-	if len(vpcStr) > 0 {
+	{
 		vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr)
 		if err != nil {
 			if err == sql.ErrNoRows {

pkg/compute/options/options.go

@@ -116,7 +116,7 @@ type ComputeOptions struct {
 
 	SyncPurgeRemovedResources []string `help:"resources that shoud be purged immediately if found removed"`
 
-	DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"24"`
+	DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"2"`
 
 	BaremetalServerReuseHostIp bool `help:"baremetal server reuse host IP address, default true" default:"true"`
 

pkg/compute/regiondrivers/aliyun.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
 
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
@@ -52,6 +53,26 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SAliyunRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SAliyunRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SAliyunRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+}
+
+func (self *SAliyunRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 1
+}
+
+func (self *SAliyunRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 100
+}
+
 func (self *SAliyunRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_ALIYUN
 }

pkg/compute/regiondrivers/aws.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -38,6 +39,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/rand"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SAwsRegionDriver struct {
@@ -49,10 +51,48 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SAwsRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SAwsRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SAwsRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+}
+
+func (self *SAwsRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SAwsRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 0
+}
+
+func (self *SAwsRegionDriver) IsOnlySupportAllowRules() bool {
+	return true
+}
+
 func (self *SAwsRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_AWS
 }
 
+func networkCheck(network *models.SNetwork) error {
+	total := network.GetPorts()
+	used, err := network.GetTotalNicCount()
+	if err != nil {
+		return errors.Wrap(err, "validateAwsLbNetwork.GetTotalNicCount")
+	}
+
+	if (total - used) < 8 {
+		return fmt.Errorf("network %s free ip is less than 8", network.GetId())
+	}
+
+	return nil
+}
+
 func validateAwsLbNetwork(ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict, requiredMin int) (*jsonutils.JSONDict, error) {
 	var networkIds []string
 	if ns, err := data.GetString("network"); err != nil {
@@ -74,21 +114,45 @@ func validateAwsLbNetwork(ownerId mcclient.IIdentityProvider, data *jsonutils.JS
 		}
 
 		network := networkV.Model.(*models.SNetwork)
+		err := networkCheck(network)
+		if err != nil {
+			return nil, errors.Wrap(err, "validateAwsLbNetwork.networkCheck")
+		}
+
 		region, zone, vpc, _, err := network.ValidateElbNetwork(nil)
 		if err != nil {
 			return nil, err
 		} else {
 			//随机选择一个子网
 			if requiredMin == 2 && len(networkIds) == 1 {
+				var nets []models.SNetwork
 				wires := models.WireManager.Query().SubQuery()
 				q := models.NetworkManager.Query().IsFalse("pending_deleted")
+				q = models.NetworkManager.FilterByOwner(q, ownerId, rbacutils.ScopeProject)
 				q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id")))
 				q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), vpc.GetId()))
 				q = q.Filter(sqlchemy.NotEquals(wires.Field("zone_id"), zone.GetId()))
-				err := q.First(secondNet)
+				err := q.All(&nets)
 				if err != nil {
 					return nil, httperrors.NewInputParameterError("required at least %d subnet.", requiredMin)
 				}
+
+				secondNetFound := false
+				for i := range nets {
+					net := nets[i]
+					err := networkCheck(&net)
+					if err != nil {
+						continue
+					}
+
+					secondNet = &net
+					secondNetFound = true
+					break
+				}
+
+				if !secondNetFound {
+					return nil, httperrors.NewInputParameterError("required at least %d subnet with at least 8 free ip.", requiredMin)
+				}
 			}
 		}
 
@@ -881,7 +945,7 @@ func (self *SAwsRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Conte
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId()); err != nil {
 				return nil, errors.Wrap(err, "AwsRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -1094,7 +1158,7 @@ func (self *SAwsRegionDriver) RequestCreateLoadbalancerListenerRule(ctx context.
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil)
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId())
 	})
 	return nil
 }
@@ -1322,7 +1386,7 @@ func (self *SAwsRegionDriver) RequestSyncLoadbalancerListener(ctx context.Contex
 		if err := iListener.Refresh(); err != nil {
 			return nil, errors.Wrap(err, "awsRegionDriver.RequestSyncLoadbalancerListener.Refresh")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil)
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId())
 	})
 	return nil
 }

pkg/compute/regiondrivers/azure.go

@@ -18,8 +18,10 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -53,3 +55,23 @@ func (self *SAzureRegionDriver) ValidateCreateLoadbalancerCertificateData(ctx co
 func (self *SAzureRegionDriver) IsSupportClassicSecurityGroup() bool {
 	return true
 }
+
+func (self *SAzureRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SAzureRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SAzureRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
+}
+
+func (self *SAzureRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 4096
+}
+
+func (self *SAzureRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 100
+}

pkg/compute/regiondrivers/base.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -235,6 +236,30 @@ func (self *SBaseRegionDriver) RequestSyncSecurityGroup(ctx context.Context, use
 	return "", fmt.Errorf("Not Implemented RequestSyncSecurityGroup")
 }
 
+func (self *SBaseRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByDesc
+}
+
+func (self *SBaseRegionDriver) IsOnlySupportAllowRules() bool {
+	return false
+}
+
+func (self *SBaseRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SBaseRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+}
+
+func (self *SBaseRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 100
+}
+
+func (self *SBaseRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 1
+}
+
 func (self *SBaseRegionDriver) ValidateCreateDBInstanceData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.SDBInstanceCreateInput, skus []models.SDBInstanceSku, network *models.SNetwork) (*api.SDBInstanceCreateInput, error) {
 	return input, nil
 }

pkg/compute/regiondrivers/ctyun.go

@@ -18,8 +18,10 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -34,6 +36,30 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SCtyunRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SCtyunRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SCtyunRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
+}
+
+func (self *SCtyunRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SCtyunRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 0
+}
+
+func (self *SCtyunRegionDriver) IsOnlySupportAllowRules() bool {
+	return true
+}
+
 func (self *SCtyunRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_CTYUN
 }

pkg/compute/regiondrivers/google.go

@@ -15,7 +15,10 @@
 package regiondrivers
 
 import (
+	"yunion.io/x/pkg/util/secrules"
+
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
 
@@ -28,6 +31,26 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SGoogleRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SGoogleRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SGoogleRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+}
+
+func (self *SGoogleRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SGoogleRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 65535
+}
+
 func (self *SGoogleRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_GOOGLE
 }

pkg/compute/regiondrivers/huawei.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
 
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
@@ -52,6 +53,30 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SHuaWeiRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SHuaWeiRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SHuaWeiRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+}
+
+func (self *SHuaWeiRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SHuaWeiRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 0
+}
+
+func (self *SHuaWeiRegionDriver) IsOnlySupportAllowRules() bool {
+	return true
+}
+
 func (self *SHuaWeiRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_HUAWEI
 }
@@ -1851,20 +1876,20 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancer(ctx context.Context,
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
 		iRegion, err := lb.GetIRegion()
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.GetIRegion")
 		}
 		params, err := lb.GetCreateLoadbalancerParams(iRegion)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.GetCreateLoadbalancerParams")
 		}
 		iLoadbalancer, err := iRegion.CreateILoadBalancer(params)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.CreateILoadBalancer")
 		}
 
 		lb.SetModelManager(models.LoadbalancerManager, lb)
 		if err := db.SetExternalId(lb, userCred, iLoadbalancer.GetGlobalId()); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.SetExternalId")
 		}
 
 		{
@@ -1873,7 +1898,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancer(ctx context.Context,
 			if len(eipId) > 0 {
 				ieip, err := iRegion.GetIEipById(eipId)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.GetIEipById")
 				}
 
 				conf := &cloudprovider.AssociateConfig{
@@ -1883,27 +1908,27 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancer(ctx context.Context,
 
 				err = ieip.Associate(conf)
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.Associate")
 				}
 
 				eip, err := db.FetchByExternalId(models.ElasticipManager, ieip.GetGlobalId())
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.FetchByExternalId")
 				}
 
 				err = eip.(*models.SElasticip).SyncWithCloudEip(ctx, userCred, lb.GetCloudprovider(), ieip, lb.GetOwnerId())
 				if err != nil {
-					return nil, err
+					return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.SyncWithCloudEip")
 				}
 			}
 		}
 
 		if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil); err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.SyncWithCloudLoadbalancer")
 		}
 		lbbgs, err := iLoadbalancer.GetILoadBalancerBackendGroups()
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.GetILoadBalancerBackendGroups")
 		}
 		if len(lbbgs) > 0 {
 			provider := lb.GetCloudprovider()
@@ -1982,7 +2007,7 @@ func (self *SHuaWeiRegionDriver) RequestSyncLoadbalancerBackend(ctx context.Cont
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestSyncLoadbalancerBackend.GetILoadbalancerBackendById")
 			}
 
-			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil)
+			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbb.GetOwnerId())
 			if err != nil {
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestSyncLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
@@ -2041,7 +2066,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Co
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId()); err != nil {
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -2104,7 +2129,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancerListenerRule(ctx conte
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, errors.Wrap(err, "huaweiRegionDriver.RequestCreateLoadbalancerListenerRule.SetExternalId")
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil)
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId())
 	})
 	return nil
 }

pkg/compute/regiondrivers/managedvirtual.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"sort"
 	"strings"
 	"time"
 
@@ -595,7 +594,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerBackend
 		if err := db.SetExternalId(lbb, userCred, iLoadbalancerBackend.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, nil)
+		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, lbbg.GetOwnerId())
 	})
 	return nil
 }
@@ -680,7 +679,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncLoadbalancerBackend(c
 			return nil, errors.Wrap(err, "regionDriver.RequestSyncLoadbalancerBackend.GetILoadbalancerBackendById")
 		}
 
-		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil)
+		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbbg.GetOwnerId())
 	})
 	return nil
 }
@@ -780,7 +779,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene
 		if err := db.SetExternalId(lblis, userCred, iListener.GetGlobalId()); err != nil {
 			return nil, errors.Wrap(err, "db.SetExternalId")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil)
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId())
 	})
 	return nil
 }
@@ -951,7 +950,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncLoadbalancerListener(
 		if err := iListener.Refresh(); err != nil {
 			return nil, errors.Wrap(err, "regionDriver.RequestSyncLoadbalancerListener.RefreshListener")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil)
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId())
 	})
 	return nil
 }
@@ -1048,7 +1047,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil)
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId())
 	})
 	return nil
 }
@@ -1432,8 +1431,11 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 			secgroup.Name = "DefaultGroup"
 		}
 		// 避免有的云不支持重名安全组
-		groupName := secgroup.Name
-		for i := 0; i < 30; i++ {
+		randomString := func(prefix string, length int) string {
+			return fmt.Sprintf("%s-%s", prefix, rand.String(length))
+		}
+		groupName := randomString(secgroup.Name, 1)
+		for i := 2; i < 30; i++ {
 			_, err := iRegion.GetISecurityGroupByName(vpc.ExternalId, groupName)
 			if err != nil {
 				if errors.Cause(err) == cloudprovider.ErrNotFound {
@@ -1443,7 +1445,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 					return "", err
 				}
 			}
-			groupName = fmt.Sprintf("%s-%d", secgroup.Name, i)
+			groupName = randomString(secgroup.Name, i)
 		}
 		conf := &cloudprovider.SecurityGroupCreateInput{
 			Name:  groupName,
@@ -1468,35 +1470,32 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 		return "", errors.Wrap(err, "db.Update")
 	}
 
-	inAllowList := secgroup.GetInAllowList()
-	outAllowList := secgroup.GetOutAllowList()
-
 	rules, err := iSecgroup.GetRules()
 	if err != nil {
 		return "", errors.Wrap(err, "iSecgroup.GetRules")
 	}
 
-	inRules := secrules.SecurityRuleSet{}
-	outRules := secrules.SecurityRuleSet{}
-	for i := 0; i < len(rules); i++ {
-		if rules[i].Direction == secrules.DIR_IN {
-			inRules = append(inRules, rules[i])
-		} else {
-			outRules = append(outRules, rules[i])
-		}
-	}
-	sort.Sort(inRules)
-	sort.Sort(outRules)
-	_inAllowList := inRules.AllowList()
-	_outAllowList := outRules.AllowList()
-	if inAllowList.Equals(_inAllowList) && outAllowList.Equals(_outAllowList) {
+	maxPriority := region.GetDriver().GetSecurityGroupRuleMaxPriority()
+	minPriority := region.GetDriver().GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := region.GetDriver().GetDefaultSecurityGroupInRule()
+	defaultOutRule := region.GetDriver().GetDefaultSecurityGroupOutRule()
+	order := region.GetDriver().GetSecurityGroupRuleOrder()
+	onlyAllowRules := region.GetDriver().IsOnlySupportAllowRules()
+
+	localRules := secrules.SecurityRuleSet(secgroup.GetSecRules(""))
+
+	common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, localRules, rules, defaultInRule, defaultOutRule, onlyAllowRules, false)
+
+	if len(inAdds) == 0 && len(inDels) == 0 && len(outAdds) == 0 && len(outDels) == 0 {
 		return cache.ExternalId, nil
 	}
 
-	err = iSecgroup.SyncRules(secgroup.GetSecRules(""))
+	err = iSecgroup.SyncRules(common, inAdds, outAdds, inDels, outDels)
 	if err != nil {
 		return "", errors.Wrap(err, "iSecgroup.SyncRules")
 	}
+
 	return cache.ExternalId, nil
 }
 

pkg/compute/regiondrivers/openstack.go

@@ -18,9 +18,11 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -35,6 +37,30 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SOpenStackRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByDesc
+}
+
+func (self *SOpenStackRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SOpenStackRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
+}
+
+func (self *SOpenStackRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SOpenStackRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 0
+}
+
+func (self *SOpenStackRegionDriver) IsOnlySupportAllowRules() bool {
+	return true
+}
+
 func (self *SOpenStackRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_OPENSTACK
 }

pkg/compute/regiondrivers/qcloud.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -45,6 +46,26 @@ func init() {
 	models.RegisterRegionDriver(&driver)
 }
 
+func (self *SQcloudRegionDriver) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder {
+	return cloudprovider.PriorityOrderByAsc
+}
+
+func (self *SQcloudRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("in:deny any")}
+}
+
+func (self *SQcloudRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
+}
+
+func (self *SQcloudRegionDriver) GetSecurityGroupRuleMaxPriority() int {
+	return 0
+}
+
+func (self *SQcloudRegionDriver) GetSecurityGroupRuleMinPriority() int {
+	return 100
+}
+
 func (self *SQcloudRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_QCLOUD
 }
@@ -305,7 +326,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Co
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId()); err != nil {
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -592,7 +613,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListener(ctx context.C
 			}
 		}
 
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil)
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId())
 	})
 	return nil
 }
@@ -686,7 +707,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListenerRule(ctx conte
 			return nil, errors.Wrap(err, "SQcloudRegionDriver.RequestCreateLoadbalancerListener.createLoadbalancerBackendGroup")
 		}
 
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil)
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId())
 	})
 	return nil
 }
@@ -754,7 +775,7 @@ func (self *SQcloudRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx cont
 			return nil, httperrors.NewInputParameterError("backend group %s(%s) belongs to loadbalancer %s instead of %s",
 				lbbg.Name, lbbg.Id, lbbg.LoadbalancerId, lblis.LoadbalancerId)
 		} else {
-			if utils.IsInStringArray(lblis.ListenerType, []string{api.LB_LISTENER_TYPE_TCP, api.LB_LISTENER_TYPE_UDP}) {
+			if lbbg != nil && utils.IsInStringArray(lblis.ListenerType, []string{api.LB_LISTENER_TYPE_TCP, api.LB_LISTENER_TYPE_UDP}) {
 				cachedLbbgs, err := lbbg.GetQcloudCachedlbbg()
 				if err != nil {
 					return nil, err
@@ -1106,7 +1127,7 @@ func (self *SQcloudRegionDriver) RequestSyncLoadbalancerBackend(ctx context.Cont
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestSyncLoadbalancerBackend.Refresh")
 			}
 
-			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil)
+			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbb.GetOwnerId())
 			if err != nil {
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestSyncLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
@@ -1332,7 +1353,7 @@ func (self *SQcloudRegionDriver) RequestSyncLoadbalancerListener(ctx context.Con
 		}
 
 		if utils.IsInStringArray(lblis.ListenerType, []string{api.LB_LISTENER_TYPE_UDP, api.LB_LISTENER_TYPE_TCP}) {
-			return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil)
+			return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId())
 		} else {
 			// http&https listener 变更不会同步到监听规则
 			return nil, nil

pkg/compute/regiondrivers/secgroup_aws_test.go

@@ -0,0 +1,63 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regiondrivers
+
+import (
+	"testing"
+
+	"yunion.io/x/pkg/util/secrules"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+func TestAwsRuleSync(t *testing.T) {
+	driver := SAwsRegionDriver{}
+	maxPriority := driver.GetSecurityGroupRuleMaxPriority()
+	minPriority := driver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := driver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := driver.GetDefaultSecurityGroupOutRule()
+	order := driver.GetSecurityGroupRuleOrder()
+	isOnlyAllowRules := driver.IsOnlySupportAllowRules()
+
+	data := []TestData{
+		{
+			Name: "Test out deny rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 1),
+			},
+			Common:  []cloudprovider.SecurityRule{},
+			InAdds:  []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 1),
+			},
+		},
+	}
+
+	for _, d := range data {
+		t.Logf("check %s", d.Name)
+		common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, d.LocalRules, d.RemoteRules, defaultInRule, defaultOutRule, isOnlyAllowRules, true)
+		check(t, "common", common, d.Common)
+		check(t, "inAdds", inAdds, d.InAdds)
+		check(t, "outAdds", outAdds, d.OutAdds)
+		check(t, "inDels", inDels, d.InDels)
+		check(t, "outDels", outDels, d.OutDels)
+	}
+}

pkg/compute/regiondrivers/secgroup_azure_test.go

@@ -0,0 +1,235 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regiondrivers
+
+import (
+	"sort"
+	"testing"
+
+	"yunion.io/x/pkg/util/secrules"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+func TestAzureRuleSync(t *testing.T) {
+	driver := SAzureRegionDriver{}
+	maxPriority := driver.GetSecurityGroupRuleMaxPriority()
+	minPriority := driver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := driver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := driver.GetDefaultSecurityGroupOutRule()
+	order := driver.GetSecurityGroupRuleOrder()
+	isOnlyAllowRules := driver.IsOnlySupportAllowRules()
+
+	data := []TestData{
+		{
+			Name:        "Test empty rules",
+			LocalRules:  secrules.SecurityRuleSet{},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 2097),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name:       "Test remove rules",
+			LocalRules: secrules.SecurityRuleSet{},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("test-name", "out:allow any", 1000),
+			},
+			Common: []cloudprovider.SecurityRule{},
+			InAdds: []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 2097),
+			},
+			InDels: []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{
+				remoteRuleWithName("test-name", "out:allow any", 1000),
+			},
+		},
+		{
+			Name: "Test diff rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:allow tcp 100-200", 99),
+				localRuleWithPriority("out:allow udp 200-300", 98),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("test-tcp", "out:allow tcp 100-200", 1000),
+				remoteRuleWithName("test-udp", "out:allow udp 200-300", 1002),
+			},
+			Common: []cloudprovider.SecurityRule{
+				remoteRuleWithName("test-tcp", "out:allow tcp 100-200", 1000),
+				remoteRuleWithName("test-udp", "out:allow udp 200-300", 1002),
+			},
+			InAdds: []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 2097),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name: "Test add rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("in:allow tcp", 100),
+				localRuleWithPriority("in:allow udp", 99),
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-ssh", "in:allow tcp 22", 300),
+			},
+			Common: []cloudprovider.SecurityRule{},
+			InAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "in:allow tcp", 2097),
+				remoteRuleWithName("", "in:allow udp", 2097),
+			},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-ssh", "in:allow tcp 22", 300),
+			},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name: "Test insert rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("in:allow tcp", 100),
+				localRuleWithPriority("in:allow udp", 99),
+				localRuleWithPriority("in:allow icmp", 98),
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-tcp", "in:allow tcp", 300),
+				remoteRuleWithName("allow-icmp", "in:allow icmp", 400),
+			},
+			Common: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-tcp", "in:allow tcp", 300),
+				remoteRuleWithName("allow-icmp", "in:allow icmp", 400),
+			},
+			InAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "in:allow udp", 2097),
+			},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name: "Test icmp rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("in:allow tcp 33", 10),
+				localRuleWithPriority("in:allow tcp 22", 1),
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-tcp-22", "in:allow tcp 22", 300),
+			},
+			Common: []cloudprovider.SecurityRule{
+				remoteRuleWithName("allow-tcp-22", "in:allow tcp 22", 300),
+			},
+			InAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "in:allow tcp 33", 299),
+			},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name: "Test a rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("in:allow tcp 1050", 5),
+				localRuleWithPriority("in:allow tcp 1011", 4),
+				localRuleWithPriority("in:allow tcp 1002", 3),
+				localRuleWithPriority("in:allow tcp 22", 2),
+				localRuleWithPriority("in:allow udp 55", 1),
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
+				remoteRuleWithName("in_allow_tcp_22_4013", "in:allow tcp 22", 4013),
+				remoteRuleWithName("in_allow_tcp_1002_4012", "in:allow tcp 1002", 4012),
+				remoteRuleWithName("in_allow_tcp_1010_4011", "in:allow tcp 1010", 4011),
+				remoteRuleWithName("in_allow_tcp_1050_4010", "in:allow tcp 1050", 4010),
+			},
+			Common: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_tcp_1050_4010", "in:allow tcp 1050", 4010),
+				remoteRuleWithName("in_allow_tcp_1002_4012", "in:allow tcp 1002", 4012),
+				remoteRuleWithName("in_allow_tcp_22_4013", "in:allow tcp 22", 4013),
+				remoteRuleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
+			},
+			InAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "in:allow tcp 1011", 4011),
+			},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_tcp_1010_4011", "in:allow tcp 1010", 4011),
+			},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+		{
+			Name: "Test b rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("in:allow udp 1055", 20),
+				localRuleWithPriority("in:allow icmp", 15),
+				localRuleWithPriority("in:allow tcp 1050", 5),
+				localRuleWithPriority("in:allow tcp 1012", 4),
+				localRuleWithPriority("in:allow tcp 1002", 3),
+				localRuleWithPriority("in:allow tcp 22", 2),
+				localRuleWithPriority("in:allow udp 55", 1),
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
+				remoteRuleWithName("in_allow_tcp_22_4013", "in:allow tcp 22", 4013),
+				remoteRuleWithName("in_allow_tcp_1002_4012", "in:allow tcp 1002", 4012),
+				remoteRuleWithName("in_allow_tcp_1012_4011", "in:allow tcp 1012", 4011),
+				remoteRuleWithName("in_allow_tcp_1050_4010", "in:allow tcp 1050", 4010),
+				remoteRuleWithName("in_allow_tcp_1055_4009", "in:allow tcp 1055", 4009),
+			},
+			Common: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_tcp_1050_4010", "in:allow tcp 1050", 4010),
+				remoteRuleWithName("in_allow_tcp_1012_4011", "in:allow tcp 1012", 4011),
+				remoteRuleWithName("in_allow_tcp_1002_4012", "in:allow tcp 1002", 4012),
+				remoteRuleWithName("in_allow_tcp_22_4013", "in:allow tcp 22", 4013),
+				remoteRuleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
+			},
+			InAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "in:allow icmp", 2097),
+				remoteRuleWithName("", "in:allow udp 1055", 4013),
+			},
+			OutAdds: []cloudprovider.SecurityRule{},
+			InDels: []cloudprovider.SecurityRule{
+				remoteRuleWithName("in_allow_tcp_1055_4009", "in:allow tcp 1055", 4009),
+			},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+	}
+
+	for _, d := range data {
+		t.Logf("check %s", d.Name)
+		common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, d.LocalRules, d.RemoteRules, defaultInRule, defaultOutRule, isOnlyAllowRules, true)
+		sort.Sort(cloudprovider.SecurityRuleSet(common))
+		sort.Sort(cloudprovider.SecurityRuleSet(inAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(outAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(inDels))
+		sort.Sort(cloudprovider.SecurityRuleSet(outDels))
+		check(t, "common", common, d.Common)
+		check(t, "inAdds", inAdds, d.InAdds)
+		check(t, "outAdds", outAdds, d.OutAdds)
+		check(t, "inDels", inDels, d.InDels)
+		check(t, "outDels", outDels, d.OutDels)
+	}
+}

pkg/compute/regiondrivers/secgroup_ctyun_test.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regiondrivers
+
+import (
+	"sort"
+	"testing"
+
+	"yunion.io/x/pkg/util/secrules"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+func TestCtyunRuleSync(t *testing.T) {
+	driver := SCtyunRegionDriver{}
+	maxPriority := driver.GetSecurityGroupRuleMaxPriority()
+	minPriority := driver.GetSecurityGroupRuleMinPriority()
+
+	defaultInRule := driver.GetDefaultSecurityGroupInRule()
+	defaultOutRule := driver.GetDefaultSecurityGroupOutRule()
+	order := driver.GetSecurityGroupRuleOrder()
+	isOnlyAllowRules := driver.IsOnlySupportAllowRules()
+
+	data := []TestData{
+		{
+			Name: "Test out deny rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:deny tcp 200", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow icmp", 0),
+				remoteRuleWithName("", "out:allow tcp 1-199", 0),
+				remoteRuleWithName("", "out:allow tcp 201-65535", 0),
+				remoteRuleWithName("", "out:allow udp", 0),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
+	}
+
+	for _, d := range data {
+		t.Logf("check %s", d.Name)
+		common, inAdds, outAdds, inDels, outDels := cloudprovider.CompareRules(minPriority, maxPriority, order, d.LocalRules, d.RemoteRules, defaultInRule, defaultOutRule, isOnlyAllowRules, true)
+		sort.Sort(cloudprovider.SecurityRuleSet(common))
+		sort.Sort(cloudprovider.SecurityRuleSet(inAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(outAdds))
+		sort.Sort(cloudprovider.SecurityRuleSet(inDels))
+		sort.Sort(cloudprovider.SecurityRuleSet(outDels))
+		check(t, "common", common, d.Common)
+		check(t, "inAdds", inAdds, d.InAdds)
+		check(t, "outAdds", outAdds, d.OutAdds)
+		check(t, "inDels", inDels, d.InDels)
+		check(t, "outDels", outDels, d.OutDels)
+	}
+}