Merge pull request #6694 from ioito/automated-cherry-pick-of-#6692-upstream-release-3.1

Automated cherry pick of #6692: fix: 返回object默认acl

cmd/climc/shell/cloudaccounts.go

@@ -26,6 +26,28 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
 
+func parseGcpCredential(filename string) (jsonutils.JSONObject, error) {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	authParams, err := jsonutils.Parse(data)
+	if err != nil {
+		return nil, err
+	}
+	ret := jsonutils.NewDict()
+	for _, k := range []string{
+		"client_email",
+		"project_id",
+		"private_key_id",
+		"private_key",
+	} {
+		v, _ := authParams.Get(k)
+		ret.Add(v, fmt.Sprintf("gcp_%s", k))
+	}
+	return ret, nil
+}
+
 func init() {
 
 	type CloudaccountListOptions struct {
@@ -138,11 +160,7 @@ func init() {
 	R(&options.SGoogleCloudAccountCreateOptions{}, "cloud-account-create-google", "Create a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountCreateOptions) error {
 		params := jsonutils.Marshal(args)
 		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Google"), "provider")
-		data, err := ioutil.ReadFile(args.GoogleJsonFile)
-		if err != nil {
-			return err
-		}
-		authParams, err := jsonutils.Parse(data)
+		authParams, err := parseGcpCredential(args.GoogleJsonFile)
 		if err != nil {
 			return err
 		}
@@ -688,11 +706,7 @@ func init() {
 	})
 
 	R(&options.SGoogleCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-google", "Update credential of a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountUpdateCredentialOptions) error {
-		data, err := ioutil.ReadFile(args.GoogleJsonFile)
-		if err != nil {
-			return err
-		}
-		params, err := jsonutils.Parse(data)
+		params, err := parseGcpCredential(args.GoogleJsonFile)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/copyright.go

@@ -15,7 +15,14 @@
 package shell
 
 import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
 
 	"yunion.io/x/jsonutils"
 
@@ -56,4 +63,78 @@ func init() {
 		printObject(r)
 		return nil
 	})
+
+	type EnterpriseUpdateOptions struct {
+		Name      string `help:"Enterprise name"`
+		Copyright string `help:"The Email"`
+		Logo      string `help:"logo file path"`
+		LoginLogo string `help:"login page logo file path"`
+		Favicon   string `help:"favicon path"`
+	}
+
+	R(&EnterpriseUpdateOptions{}, "infos-update", "update enterprise info", func(s *mcclient.ClientSession, args *EnterpriseUpdateOptions) error {
+		var b bytes.Buffer
+		w := multipart.NewWriter(&b)
+		if len(args.Name) > 0 {
+			w.WriteField("name", args.Name)
+		}
+
+		if len(args.Copyright) > 0 {
+			w.WriteField("copyright", args.Copyright)
+		}
+
+		writeFile := func(filedName, filePath string) error {
+			if len(filePath) == 0 {
+				return nil
+			}
+
+			c, err := ioutil.ReadFile(filePath)
+			if err != nil {
+				return err
+			}
+
+			var v string
+			c64 := base64.StdEncoding.EncodeToString(c)
+			ext := filepath.Ext(filePath)
+			switch ext {
+			case ".png":
+				v = fmt.Sprintf("data:image/png;base64,%s", c64)
+			case ".jpg":
+				v = fmt.Sprintf("data:image/jpg;base64,%s", c64)
+			default:
+				return fmt.Errorf("only support png/jpg image")
+			}
+
+			err = w.WriteField(filedName, v)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		if err := writeFile("logo", args.Logo); err != nil {
+			return err
+		}
+
+		if err := writeFile("login_logo", args.LoginLogo); err != nil {
+			return err
+		}
+
+		if err := writeFile("favicon", args.Favicon); err != nil {
+			return err
+		}
+
+		header := make(http.Header)
+		header.Set("content-type", fmt.Sprintf("multipart/form-data; boundary=%s", w.Boundary()))
+		w.Close()
+
+		ret, err := modules.Info.Update(s, header, bufio.NewReader(&b))
+		if err != nil {
+			return err
+		}
+
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/k8s/event.go

@@ -0,0 +1,9 @@
+package k8s
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+)
+
+func initEvent() {
+	initK8sNamespaceResource("event", k8s.Events)
+}

cmd/climc/shell/k8s/k8s.go

@@ -16,6 +16,11 @@ package k8s
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+
+	"github.com/ghodss/yaml"
 
 	"yunion.io/x/jsonutils"
 
@@ -57,7 +62,7 @@ func init() {
 	initPVC()
 	initJob()
 	initCronJob()
-
+	initEvent()
 	initRbac()
 
 	initApp()
@@ -217,6 +222,71 @@ func NewK8sNsResourceGetCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cm
 	)
 }
 
+func NewK8sNsResourceGetRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("show-raw"),
+		fmt.Sprintf("Show k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			ret, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			printObjectYAML(ret)
+			return nil
+		},
+	)
+}
+
+func NewK8sResourceEditRawCmd(cmdN CmdNameFactory, manager k8s.IClusterResourceManager) *Cmd {
+	return NewCommand(
+		&o.NamespaceResourceGetOptions{},
+		cmdN.Do("edit-raw"),
+		fmt.Sprintf("Edit and update k8s %s raw data", cmdN.Kind),
+		func(s *mcclient.ClientSession, args *o.NamespaceResourceGetOptions) error {
+			rawData, err := manager.GetRaw(s, args.NAME, args.Params())
+			if err != nil {
+				return err
+			}
+			yamlBytes := rawData.YAMLString()
+			tempfile, err := ioutil.TempFile("", fmt.Sprintf("k8s-%s-%s*.yaml", cmdN.Kind, args.NAME))
+			if err != nil {
+				return err
+			}
+			defer os.Remove(tempfile.Name())
+			if _, err := tempfile.Write([]byte(yamlBytes)); err != nil {
+				return err
+			}
+			if err := tempfile.Close(); err != nil {
+				return err
+			}
+
+			cmd := exec.Command("vim", tempfile.Name())
+			cmd.Stdin = os.Stdin
+			cmd.Stdout = os.Stdout
+			if err := cmd.Run(); err != nil {
+				return err
+			}
+			content, err := ioutil.ReadFile(tempfile.Name())
+			if err != nil {
+				return err
+			}
+			jsonBytes, err := yaml.YAMLToJSON(content)
+			if err != nil {
+				return err
+			}
+			body, err := jsonutils.Parse(jsonBytes)
+			if err != nil {
+				return err
+			}
+			if _, err := manager.UpdateRaw(s, args.NAME, args.Params(), body.(*jsonutils.JSONDict)); err != nil {
+				return err
+			}
+			return nil
+		},
+	)
+}
+
 func NewK8sResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager) *Cmd {
 	return NewCommand(
 		&o.ResourceDeleteOptions{},
@@ -244,11 +314,13 @@ func NewK8sNsResourceDeleteCmd(cmdN CmdNameFactory, manager modulebase.Manager)
 	return deleteCmd
 }
 
-func initK8sNamespaceResource(kind string, manager modulebase.Manager) *ShellCommands {
+func initK8sNamespaceResource(kind string, manager k8s.IClusterResourceManager) *ShellCommands {
 	cmdN := NewCmdNameFactory(kind)
 	return NewShellCommands(cmdN.Do).AddR(
 		NewK8sNsResourceListCmd(cmdN, manager),
 		NewK8sNsResourceGetCmd(cmdN, manager),
 		NewK8sNsResourceDeleteCmd(cmdN, manager),
+		NewK8sNsResourceGetRawCmd(cmdN, manager),
+		NewK8sResourceEditRawCmd(cmdN, manager),
 	)
 }

cmd/climc/shell/policies.go

@@ -375,7 +375,7 @@ func init() {
 			rbacutils.ScopeUser,
 			rbacutils.ScopeNone,
 		} {
-			m := policy.PolicyManager.MatchedPolicies(scope, token)
+			m := policy.PolicyManager.MatchedPolicyNames(scope, token)
 			fmt.Println("matched", scope, "policies:", m)
 		}
 

cmd/climc/shell/servers.go

@@ -988,6 +988,7 @@ func init() {
 				config.Hosts[i].HostIp = yamlConfig.Hosts[i].HostIp
 				config.Hosts[i].XmlFilePath = yamlConfig.Hosts[i].XmlFilePath
 				config.Hosts[i].Servers = make([]compute.SLibvirtServerConfig, len(yamlConfig.Hosts[i].Servers))
+				config.Hosts[i].MonitorPath = yamlConfig.Hosts[i].MonitorPath
 				for j := 0; j < len(yamlConfig.Hosts[i].Servers); j++ {
 					config.Hosts[i].Servers[j].MacIp = make(map[string]string)
 					mac := yamlConfig.Hosts[i].Servers[j].Mac
@@ -1010,11 +1011,11 @@ func init() {
 		if err != nil {
 			return err
 		}
-		for i := 0; i < len(params); i++ {
-			val := jsonutils.NewDict()
-			val.Set(modules.Servers.KeywordPlural, params[i])
-			params[i] = val
-		}
+		//for i := 0; i < len(params); i++ {
+		//	val := jsonutils.NewDict()
+		//	val.Set(modules.Servers.KeywordPlural, params[i])
+		//	params[i] = val
+		//}
 
 		results := modules.Servers.BatchPerformClassAction(s, "import-from-libvirt", params)
 		printBatchResults(results, modules.Servers.GetColumns(s))

go.mod

@@ -3,7 +3,6 @@ module yunion.io/x/onecloud
 go 1.12
 
 require (
-	cloud.google.com/go v0.38.0
 	github.com/360EntSecGroup-Skylar/excelize v1.4.0
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible
 	github.com/Azure/go-autorest/autorest v0.9.3
@@ -115,7 +114,6 @@ require (
 	golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
-	google.golang.org/api v0.13.0 // indirect
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
 	google.golang.org/grpc v1.23.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
@@ -129,12 +127,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b
+	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
-	yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624
+	yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
-	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
+	yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e
 )
 
 replace (

go.sum

@@ -371,8 +371,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
@@ -897,8 +895,6 @@ gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmK
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
-google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
@@ -1015,8 +1011,8 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b h1:mt0TOKRk76yeH0whJfmKsceXBuudXLjvoj8NKGTqpEU=
-yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501 h1:i1r9XvbdxH3FgTCLmTaRi3MzQqhiQimXJRlUOPgrxnU=
+yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
@@ -1025,11 +1021,11 @@ yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624 h1:yPayJlTJHOP0vKY6ovliZyTwJY9RvdWEF+QDqnxt/iw=
-yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa h1:VizPfW8+mLFEE7W/97zxQJbfdxchi2dn1M/Y7YBwTTc=
+yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
-yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e h1:pctCe/EPel3F1B83pJ2q9b34Umd1NdbLW1Yd+Lzur2s=
+yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -570,6 +570,7 @@ func (this *projectRoles) add(roleId, roleName string) {
 
 func (this *projectRoles) getToken(scope rbacutils.TRbacScope, user, userId, domain, domainId string, ip string) mcclient.TokenCredential {
 	return &mcclient.SSimpleToken{
+		Token:           "faketoken",
 		Domain:          domain,
 		DomainId:        domainId,
 		User:            user,
@@ -609,12 +610,12 @@ func (this *projectRoles) json(user, userId, domain, domainId string, ip string)
 	}
 	obj.Add(roles, "roles")
 	for _, scope := range []rbacutils.TRbacScope{
-		rbacutils.ScopeSystem,
-		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeSystem,
 	} {
 		token := this.getToken(scope, user, userId, domain, domainId, ip)
-		matches := policy.PolicyManager.MatchedPolicies(scope, token)
+		matches := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		obj.Add(jsonutils.NewStringArray(matches), fmt.Sprintf("%s_policies", scope))
 		if len(matches) > 0 {
 			obj.Add(jsonutils.JSONTrue, fmt.Sprintf("%s_capable", scope))
@@ -741,7 +742,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		rbacutils.ScopeDomain,
 		rbacutils.ScopeProject,
 	} {
-		p := policy.PolicyManager.MatchedPolicies(scope, token)
+		p := policy.PolicyManager.MatchedPolicyNames(scope, token)
 		data.Add(jsonutils.NewStringArray(p), fmt.Sprintf("%s_policies", scope))
 		if scope == rbacutils.ScopeSystem {
 			data.Add(jsonutils.NewStringArray(p), "admin_policies")

pkg/apigateway/handler/init.go

@@ -1,4 +1,4 @@
-// Copyright 2018, OpenCensus Authors
+// Copyright 2019 Yunion
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,12 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !go1.9
+package handler
 
-package tag
-
-import "context"
-
-func do(ctx context.Context, f func(ctx context.Context)) {
-	f(ctx)
-}
+import (
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
+)

pkg/apigateway/handler/resource.go

@@ -606,28 +606,45 @@ func (f *ResourceHandlers) patchJointHandler(ctx context.Context, w http.Respons
 	}
 }
 
-// batch update Joint
+// * batch update Joint
+// * put specific
+// /<resname>/<resid>/<resname2>
+// /<resname>/<resid>/<spec>
 func (f *ResourceHandlers) batchUpdateJointHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	req := newRequest(ctx, w, r).WithMod1().WithMod2()
+	req := newRequest(ctx, w, r).WithMod1()
 	if err := req.Error(); err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
 	}
 	session := req.Session()
 	module := req.Mod1()
-	module2 := req.Mod2()
 	body := req.Body()
 	query := req.Query()
 
-	idlist := fetchIdList(query, w)
-	if idlist == nil {
+	if idlist, _ := query.GetArray("id"); len(idlist) == 0 {
+		// do put specific
+		spec := req.ResName2()
+		obj, e := module.PutSpecific(session, req.ResID(), spec, query, body)
+		if e != nil {
+			httperrors.GeneralServerError(w, e)
+		} else {
+			appsrv.SendJSON(w, obj)
+		}
 		return
 	}
+
+	req = req.WithMod2()
+	if err := req.Error(); err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	module2 := req.Mod2()
 	jmod, e := modulebase.GetJointModule2(session, module, module2)
 	if e != nil { // update joint
 		httperrors.GeneralServerError(w, e)
 		return
 	}
+	idlist := fetchIdList(query, w)
 	ret := jmod.BatchUpdate(session, req.ResID(), idlist, query, body)
 	w.WriteHeader(207)
 	appsrv.SendJSON(w, modulebase.SubmitResults2JSON(ret))

pkg/apis/cloudcommon/proxy/proxy.go

@@ -42,7 +42,11 @@ func (ps *SProxySetting) IsZero() bool {
 
 type ProxySettingResourceInput struct {
 	// 代理配置
-	ProxySettingId string `json:"proxy_setting_id"`
+	ProxySetting string `json:"proxy_setting"`
+
+	// swagger:ignore
+	// Deprecated
+	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
 }
 
 type ProxySettingTestInput struct {

pkg/apis/cloudevent/cloudevent.go

@@ -18,13 +18,59 @@ import (
 	"time"
 
 	"yunion.io/x/onecloud/pkg/apis"
-	"yunion.io/x/onecloud/pkg/apis/compute"
 )
 
 type CloudeventListInput struct {
-	apis.VirtualResourceListInput
+	apis.ModelBaseListInput
 
-	compute.ManagedResourceListInput
+	// 列出指定云平台的资源，支持的云平台如下
+	//
+	// | Provider  | 开始支持版本 | 平台                                |
+	// |-----------|------------|-------------------------------------|
+	// | OneCloud  | 0.0        | OneCloud内置私有云，包括KVM和裸金属管理 |
+	// | VMware    | 1.2        | VMware vCenter                      |
+	// | OpenStack | 2.6        | OpenStack M版本以上私有云             |
+	// | ZStack    | 2.10       | ZStack私有云                         |
+	// | Aliyun    | 2.0        | 阿里云                               |
+	// | Aws       | 2.3        | Amazon AWS                          |
+	// | Azure     | 2.2        | Microsoft Azure                     |
+	// | Google    | 2.13       | Google Cloud Platform               |
+	// | Qcloud    | 2.3        | 腾讯云                               |
+	// | Huawei    | 2.5        | 华为公有云                           |
+	// | Ucloud    | 2.7        | UCLOUD                               |
+	// | Ctyun     | 2.13       | 天翼云                               |
+	// | S3        | 2.11       | 通用s3对象存储                        |
+	// | Ceph      | 2.11       | Ceph对象存储                         |
+	// | Xsky      | 2.11       | XSKY启明星辰Ceph对象存储              |
+	//
+	// enum: OneCloud,VMware,Aliyun,Qcloud,Azure,Aws,Huawei,OpenStack,Ucloud,ZStack,Google,Ctyun,S3,Ceph,Xsky"
+	Providers []string `json:"providers"`
+	// swagger:ignore
+	// Deprecated
+	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
+
+	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
+	//
+	// |   Brand  | Provider | 说明        |
+	// |----------|----------|------------|
+	// | DStack   | ZStack   | 滴滴云私有云 |
+	//
+	Brands []string `json:"brands"`
+	// swagger:ignore
+	// Deprecated
+	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
+
+	// 服务类型
+	Service []string `json:"service"`
+
+	// 订阅
+	Manager []string `json:"manager"`
+
+	// 账号
+	Account []string `json:"account"`
+
+	// 操作类型
+	Action []string `json:"action"`
 
 	// 操作日志起始时间
 	Since time.Time `json:"since"`

pkg/apis/compute/api.go

@@ -285,7 +285,7 @@ type ServerConfigs struct {
 	// swagger:ignore
 	// Deprecated
 	// alias for InstanceType
-	Sku string `json:"sku" deprecated-by:"instance_type"`
+	Sku string `json:"sku" "yunion:deprecated-by":"instance_type"`
 
 	// 虚拟机高可用(创建备机)
 	// default: false
@@ -357,12 +357,13 @@ type ServerCreateInput struct {
 	// required: false
 	UserData string `json:"user_data"`
 
+	// Deprecated
 	// swagger:ignore
-	Keypair string `json:"keypair" deprecated-by:"keypair_id"`
+	KeypairId string `json:"keypair_id" "yunion:deprecated-by":"keypair"`
 
 	// 秘钥对Id
 	// required: false
-	KeypairId string `json:"keypair_id"`
+	Keypair string `json:"keypair"`
 
 	// 密码
 	// 要求: 密码长度 >= 20, 至少包含一个数字一个小写字母一个大小字母及特殊字符~`!@#$%^&*()-_=+[]{}|:';\",./<>?中的一个

pkg/apis/compute/cloudprovider.go

@@ -147,30 +147,30 @@ type ManagedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Manager string `json:"manager" deprecated-by:"cloudprovider"`
+	Manager string `json:"manager" "yunion:deprecated-by":"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	ManagerId string `json:"manager_id" deprecated-by:"cloudprovider"`
+	ManagerId string `json:"manager_id" "yunion:deprecated-by":"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudproviderId string `json:"cloudprovider_id" deprecated-by:"cloudprovider"`
+	CloudproviderId string `json:"cloudprovider_id" "yunion:deprecated-by":"cloudprovider"`
 
 	// 列出关联指定云账号(ID或Name)的资源
 	Cloudaccount string `json:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudaccountId string `json:"cloudaccount_id" deprecated-by:"cloudaccount"`
+	CloudaccountId string `json:"cloudaccount_id" "yunion:deprecated-by":"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Account string `json:"account" deprecated-by:"cloudaccount"`
+	Account string `json:"account" "yunion:deprecated-by":"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	AccountId string `json:"account_id" deprecated-by:"cloudaccount"`
+	AccountId string `json:"account_id" "yunion:deprecated-by":"cloudaccount"`
 
 	// 列出指定云平台的资源，支持的云平台如下
 	//
@@ -196,7 +196,7 @@ type ManagedResourceListInput struct {
 	Providers []string `json:"providers"`
 	// swagger:ignore
 	// Deprecated
-	Provider []string `json:"provider" deprecated-by:"providers"`
+	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
 
 	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
 	//
@@ -207,7 +207,7 @@ type ManagedResourceListInput struct {
 	Brands []string `json:"brands"`
 	// swagger:ignore
 	// Deprecated
-	Brand []string `json:"brand" deprecated-by:"brands"`
+	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
 
 	// 列出指定云环境的资源，支持云环境如下：
 	//

pkg/apis/compute/cloudregions_const.go

@@ -56,6 +56,7 @@ const (
 	CITY_NAN_JING      = "Nanjing"     //南京
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
+	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京
@@ -111,6 +112,7 @@ const (
 	CITY_US_GOV_WEST    = "us-gov-west"    //???
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
+	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -149,6 +151,12 @@ const (
 	// 巴西
 	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
 
+	// 智利
+	CITY_SANTIAGO = "Santiago" // 圣地亚哥
+
+	// 墨西哥
+	CITY_MEXICO = "Mexico" // 墨西哥
+
 	// 荷兰
 	CITY_HOLLAND = "Holland" //荷兰
 
@@ -194,4 +202,6 @@ const (
 	COUNTRY_CODE_VN = "VN" //越南
 	COUNTRY_CODE_CH = "CH" //瑞士
 	COUNTRY_CODE_NO = "NO" //挪威
+	COUNTRY_CODE_MX = "MX" //墨西哥
+	COUNTRY_CODE_CL = "CL" //智利
 )

pkg/apis/compute/dbinstance.go

@@ -212,7 +212,7 @@ type DbinstanceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by dbinstance_id
-	DbinstanceId string `json:"dbinstance_id" deprecated-by:"dbinstance"`
+	DbinstanceId string `json:"dbinstance_id" "yunion:deprecated-by":"dbinstance"`
 }
 
 type DBInstanceDetails struct {

pkg/apis/compute/disk.go

@@ -89,7 +89,7 @@ type SnapshotPolicyFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter disk by snapshotpolicy_id
-	SnapshotpolicyId string `json:"snapshotpolicy_id" deprecated-by:"snapshotpolicy"`
+	SnapshotpolicyId string `json:"snapshotpolicy_id" "yunion:deprecated-by":"snapshotpolicy"`
 }
 
 type DiskListInput struct {
@@ -109,7 +109,7 @@ type DiskListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk type
-	Type string `json:"type" deprecated-by:"disk_type"`
+	Type string `json:"type" "yunion:deprecated-by":"disk_type"`
 	// 过滤指定disk_type的磁盘列表，可能的值为：sys, data, swap. volume
 	//
 	// | disk_type值 | 说明 |
@@ -128,7 +128,7 @@ type DiskFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk_id
-	DiskId string `json:"disk_id" deprecated-by:"disk"`
+	DiskId string `json:"disk_id" "yunion:deprecated-by":"disk"`
 }
 
 type SimpleGuest struct {

pkg/apis/compute/geo_input.go

@@ -27,15 +27,15 @@ type RegionalFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudregionId string `json:"cloudregion_id" deprecated-by:"cloudregion"`
+	CloudregionId string `json:"cloudregion_id" "yunion:deprecated-by":"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Region string `json:"region" deprecated-by:"cloudregion"`
+	Region string `json:"region" "yunion:deprecated-by":"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	RegionId string `json:"region_id" deprecated-by:"cloudregion"`
+	RegionId string `json:"region_id" "yunion:deprecated-by":"cloudregion"`
 }
 
 type ZonalFilterListInput struct {
@@ -46,7 +46,7 @@ type ZonalFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by zone_id
-	ZoneId string `json:"zone_id" deprecated-by:"zone"`
+	ZoneId string `json:"zone_id" "yunion:deprecated-by":"zone"`
 	// 过滤处于多个指定可用区内的资源
 	Zones []string `json:"zones"`
 }
@@ -67,7 +67,7 @@ type HostFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by host_id
-	HostId string `json:"host_id" deprecated-by:"host"`
+	HostId string `json:"host_id" "yunion:deprecated-by":"host"`
 }
 
 type CloudregionListInput struct {

pkg/apis/compute/guests.go

@@ -28,15 +28,15 @@ type ServerFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	ServerId string `json:"server_id" deprecated-by:"server"`
+	ServerId string `json:"server_id" "yunion:deprecated-by":"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	Guest string `json:"guest" deprecated-by:"server"`
+	Guest string `json:"guest" "yunion:deprecated-by":"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	GuestId string `json:"guest_id" deprecated-by:"server"`
+	GuestId string `json:"guest_id" "yunion:deprecated-by":"server"`
 }
 
 type ServerListInput struct {

pkg/apis/compute/instance_group.go

@@ -41,5 +41,5 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// deprecated: true
 	// Filter by instance group Id
-	GroupId string `json:"group_id" deprecated-by:"group"`
+	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
 }

pkg/apis/compute/network.go

@@ -24,7 +24,7 @@ type VpcFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by vpc Id
-	VpcId string `json:"vpc_id" deprecated-by:"vpc"`
+	VpcId string `json:"vpc_id" "yunion:deprecated-by":"vpc"`
 }
 
 type WireFilterListInput struct {
@@ -35,7 +35,7 @@ type WireFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// fitler by wire id
-	WireId string `json:"wire_id" deprecated-by:"wire"`
+	WireId string `json:"wire_id" "yunion:deprecated-by":"wire"`
 }
 
 type NetworkFilterListInput struct {
@@ -46,7 +46,7 @@ type NetworkFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by networkId
-	NetworkId string `json:"network_id" deprecated-by:"network"`
+	NetworkId string `json:"network_id" "yunion:deprecated-by":"network"`
 }
 
 type NetworkListInput struct {
@@ -60,9 +60,13 @@ type NetworkListInput struct {
 
 	WireFilterListInput
 
-	// description: search ip address in network.
+	// description: Exact matching ip address in network.
 	// example: 10.168.222.1
 	Ip string `json:"ip"`
+
+	// description: Fuzzy matching ip address in network.
+	// example: 10.168.222.1
+	IpMatch string `json:"ip_match"`
 }
 
 type NetworkCreateInput struct {

pkg/apis/compute/schedtag.go

@@ -45,7 +45,7 @@ type SchedtagFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by schedtag_id
-	SchedtagId string `json:"schedtag_id" deprecated-by:"schedtag"`
+	SchedtagId string `json:"schedtag_id" "yunion:deprecated-by":"schedtag"`
 }
 
 type SchedtagListInput struct {
@@ -56,7 +56,7 @@ type SchedtagListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by type, alias for resource_type
-	Type string `json:"type" deprecated-by:"resource_type"`
+	Type string `json:"type" "yunion:deprecated-by":"resource_type"`
 }
 
 type SchedtagDetails struct {

pkg/apis/compute/secgroup.go

@@ -162,7 +162,7 @@ type SecgroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by secgroup_id
-	SecgroupId string `json:"secgroup_id" deprecated-by:"secgroup"`
+	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
 }
 
 type SecgroupDetails struct {

pkg/apis/compute/storage_const.go

@@ -144,7 +144,7 @@ type StorageFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by storage_id
-	StorageId string `json:"storage_id" deprecated-by:"storage"`
+	StorageId string `json:"storage_id" "yunion:deprecated-by":"storage"`
 }
 
 type StorageShareFilterListInput struct {

pkg/apis/compute/wire.go

@@ -53,7 +53,7 @@ type WireDetails struct {
 	// example: 1
 	Networks int `json:"networks"`
 	// VPC名称
-	Vpc string `json:"vpc'`
+	Vpc string `json:"vpc"`
 	// VPC外部Id
 	VpcExtId string `json:"vpc_ext_id"`
 

pkg/apis/identity/assignments.go

@@ -62,3 +62,7 @@ func (ra *SRoleAssignment) GetRoles() []string {
 func (ra *SRoleAssignment) GetLoginIp() string {
 	return ""
 }
+
+func (ra *SRoleAssignment) GetTokenString() string {
+	return "faketoken"
+}

pkg/apis/identity/consts.go

@@ -144,6 +144,10 @@ var (
 			"etcd_cacert",
 			"etcd_cert",
 			"etcd_key",
+
+			"bootstrap_admin_user_password",
+			"reset_admin_user_password",
+			"fernet_key_repository",
 		},
 	}
 )

pkg/apis/identity/identityprovider.go

@@ -40,10 +40,10 @@ type IdentityProviderCreateInput struct {
 	Template string `json:"template"`
 
 	// 默认导入用户和组的域
-	TargetDomainId string `json:"target_domain_id"`
+	TargetDomain string `json:"target_domain"`
 	// swagger:ignore
 	// Deprecated
-	TargetDomain string `json:"target_domain" "yunion:deprecated-by":"target_domain_id"`
+	TargetDomainId string `json:"target_domain_id" "yunion:deprecated-by":"target_domain"`
 
 	// 新建域的时候是否自动新建第一个项目
 	AutoCreateProject *bool `json:"auto_create_project"`

pkg/apis/identity/input.go

@@ -48,15 +48,15 @@ type ProjectFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by project_id
-	ProjectId string `json:"project_id" deprecated-by:"project"`
+	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant
-	Tenant string `json:"tenant" deprecated-by:"project"`
+	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant_id
-	TenantId string `json:"tenant_id" deprecated-by:"project"`
+	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
 }
 
 type UserFilterListInput struct {
@@ -65,7 +65,7 @@ type UserFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by user_id
-	UserId string `json:"user_id" deprecated-by:"user"`
+	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
 }
 
 type GroupFilterListInput struct {
@@ -74,7 +74,7 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by group_id
-	GroupId string `json:"group_id" deprecated-by:"group"`
+	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
 }
 
 type RoleFilterListInput struct {
@@ -83,7 +83,7 @@ type RoleFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by role_id
-	RoleId string `json:"role_id" deprecated-by:"role"`
+	RoleId string `json:"role_id" "yunion:deprecated-by":"role"`
 }
 
 type ServiceFilterListInput struct {
@@ -92,7 +92,7 @@ type ServiceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by service_id
-	ServiceId string `json:"service_id" deprecated-by:"service"`
+	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
 }
 
 type RoleListInput struct {

pkg/apis/input.go

@@ -28,11 +28,11 @@ type DomainizedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Project domain Id filter, alias for project_domain
-	ProjectDomainId string `json:"project_domain_id" deprecated-by:"project_domain"`
+	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Domain Id filter, alias for project_domain
-	DomainId string `json:"domain_id" deprecated-by:"project_domain"`
+	DomainId string `json:"domain_id" "yunion:deprecated-by":"project_domain"`
 }
 
 type DomainizedResourceCreateInput struct {
@@ -53,15 +53,15 @@ type ProjectizedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by project_id, alias for project
-	ProjectId string `json:"project_id" deprecated-by:"project"`
+	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant ID or Name, alias for project
-	Tenant string `json:"tenant" deprecated-by:"project"`
+	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant_id, alias for project
-	TenantId string `json:"tenant_id" deprecated-by:"project"`
+	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
 }
 
 type ProjectizedResourceCreateInput struct {
@@ -82,7 +82,7 @@ type UserResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by userId
-	UserId string `json:"user_id" deprecated-by:"user"`
+	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
 }
 
 func (input UserResourceListInput) UserStr() string {

pkg/appsrv/workers.go

@@ -83,9 +83,6 @@ func (worker *SWorker) run() {
 			task := req.(*sWorkerTask)
 			if task.worker != nil {
 				task.worker <- worker
-				// worker channel is buffered
-				// close the worker channel
-				close(task.worker)
 			}
 			execCallback(task)
 		} else {

pkg/baremetal/handler/handlers.go

@@ -163,12 +163,7 @@ func handleBaremetalCdromTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
 }
 
 func handleBaremetalJnlpTask(ctx *Context, bm *baremetal.SBaremetalInstance) {
-	cli := bm.GetRedfishCli(ctx)
-	if cli == nil {
-		ctx.ResponseError(httperrors.NewNotSupportedError("Redfish API"))
-		return
-	}
-	jnlp, err := cli.GetConsoleJNLP(ctx)
+	jnlp, err := bm.GetConsoleJNLP(ctx)
 	if err != nil {
 		ctx.ResponseError(errors.Wrap(err, "GetConsoleJNLP"))
 		return

pkg/baremetal/manager.go

@@ -64,6 +64,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
+	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
@@ -1935,6 +1936,25 @@ func (b *SBaremetalInstance) fetchPowerThermalMetrics(ctx context.Context) ([]in
 	return powerMetrics, thermalMetrics, nil
 }
 
+func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error) {
+	cli := b.GetRedfishCli(ctx)
+	if cli != nil {
+		return cli.GetConsoleJNLP(ctx)
+	}
+	conf := b.GetIPMIConfig()
+	bmc := bmconsole.NewBMCConsole(conf.IpAddr, conf.Username, conf.Password, false)
+	manufacture := b.GetManufacture()
+	switch strings.ToLower(manufacture) {
+	case "hp", "hpe":
+		return bmc.GetIloConsoleJNLP(ctx)
+	case "dell", "dell inc.":
+		return bmc.GetIdracConsoleJNLP(ctx, "", "")
+	case "supermicro":
+		return bmc.GetSupermicroConsoleJNLP(ctx)
+	}
+	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
+}
+
 func (b *SBaremetalInstance) getTags() []influxdb.SKeyValue {
 	tags := []influxdb.SKeyValue{
 		{

pkg/baremetal/profiles/profiles.go

@@ -43,7 +43,7 @@ func InspurProfile() IPMIProfile {
 
 func LenovoProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{8},
+		LanChannel: []int{1, 8},
 		RootName:   "root",
 		RootId:     2,
 	}
@@ -51,7 +51,7 @@ func LenovoProfile() IPMIProfile {
 
 func HpProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{2},
+		LanChannel: []int{1, 2},
 		RootName:   "root",
 		RootId:     1,
 	}

pkg/baremetal/service/service.go

@@ -64,6 +64,7 @@ func (s *BaremetalService) StartService() {
 
 	fsdriver.Init(nil)
 	app := app_common.InitApp(&o.Options.BaseOptions, false)
+
 	handler.InitHandlers(app)
 
 	s.startAgent(app)

pkg/baremetal/tasks/base.go

@@ -306,7 +306,13 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 	if err != nil {
 		return errors.Wrapf(err, "Get power status")
 	}
+	maxTries := 10
+	count := 0
 	for status == "" || status == types.POWER_STATUS_OFF {
+		if count > maxTries {
+			break
+		}
+		log.Infof("Try power on %d times, pxe boot %v", count+1, self.PxeBoot)
 		if status == types.POWER_STATUS_OFF {
 			if self.PxeBoot {
 				err = self.Baremetal.DoPXEBoot()
@@ -314,7 +320,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				err = self.Baremetal.DoRedfishPowerOn()
 			}
 			if err != nil {
-				return errors.Wrapf(err, "Do boot power on")
+				log.Warningf("Do boot power on error: %v", err)
 			}
 		}
 		status, err = self.Baremetal.GetPowerStatus()
@@ -328,6 +334,7 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 				return err
 			}
 		}
+		count++
 	}
 	if status != types.POWER_STATUS_ON {
 		return fmt.Errorf("Baremetal invalid restart status: %s", status)

pkg/cloudcommon/cmdline/helper.go

@@ -402,7 +402,7 @@ func FetchServerCreateInputByJSON(obj jsonutils.JSONObject) (*compute.ServerCrea
 		input.InstanceType = skuName
 	}
 	if keypair := jsonutils.GetAnyString(obj, []string{"keypair", "keypair_id"}); keypair != "" {
-		input.KeypairId = keypair
+		input.Keypair = keypair
 	}
 	if secgroup := jsonutils.GetAnyString(obj, []string{"secgroup", "secgroup_id", "secgrp_id"}); len(secgroup) != 0 {
 		input.SecgroupId = secgroup

pkg/cloudcommon/database.go

@@ -44,7 +44,7 @@ func InitDB(options *common_options.DBOptions) {
 
 	dialect, sqlStr, err := options.GetDBConnection()
 	if err != nil {
-		log.Fatalf("Invalid SqlConnection string: %s", options.SqlConnection)
+		log.Fatalf("Invalid SqlConnection string: %s error: %v", options.SqlConnection, err)
 	}
 	dbConn, err := sql.Open(dialect, sqlStr)
 	if err != nil {

pkg/cloudcommon/db/db_dispatcher.go

@@ -686,7 +686,7 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 			}
 		}
 		// do limit
-		if limit > 0 && total > limit {
+		if limit > 0 && total-offset > limit {
 			data = data[:limit]
 		}
 	}
@@ -1155,6 +1155,12 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 	model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId)
 	if err != nil {
 		// log.Errorf("fail to doCreateItem %s", err)
+		if CancelPendingUsagesInContext != nil {
+			err := CancelPendingUsagesInContext(ctx, userCred)
+			if err != nil {
+				log.Errorf("CancelPendingUsagesInContext fail %s", err)
+			}
+		}
 		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
 		if failErr != nil {
 			log.Errorf("manager.OnCreateFailed %s", failErr)
@@ -1347,6 +1353,19 @@ func managerPerformCheckCreateData(
 		return nil, httperrors.NewForbiddenError("not allow to perform %s", action)
 	}
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+
+		defer func() {
+			if CancelPendingUsagesInContext != nil {
+				err := CancelPendingUsagesInContext(ctx, userCred)
+				if err != nil {
+					log.Errorf("CancelPendingUsagesInContext fail %s", err)
+				}
+			}
+		}()
+	}
+
 	return ValidateCreateData(manager, ctx, userCred, ownerId, query, bodyDict)
 }
 

pkg/cloudcommon/db/modelbase.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -391,12 +390,6 @@ func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Cont
 }
 
 func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	if CancelPendingUsagesInContext != nil {
-		err := CancelPendingUsagesInContext(ctx, userCred)
-		if err != nil {
-			return errors.Wrap(err, "CancelPendingUsagesInContext")
-		}
-	}
 	return nil
 }
 

pkg/cloudcommon/db/proxy/proxysetting.go

@@ -205,14 +205,14 @@ func RegisterReferrer(man db.IModelManager) {
 }
 
 func ValidateProxySettingResourceInput(userCred mcclient.TokenCredential, input proxyapi.ProxySettingResourceInput) (*SProxySetting, proxyapi.ProxySettingResourceInput, error) {
-	m, err := ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId)
+	m, err := ProxySettingManager.FetchByIdOrName(userCred, input.ProxySetting)
 	if err != nil {
 		if errors.Cause(err) == sql.ErrNoRows {
-			return nil, input, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", ProxySettingManager.Keyword(), input.ProxySettingId)
+			return nil, input, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", ProxySettingManager.Keyword(), input.ProxySetting)
 		} else {
 			return nil, input, errors.Wrapf(err, "ProxySettingManager.FetchByIdOrName")
 		}
 	}
-	input.ProxySettingId = m.GetId()
+	input.ProxySetting = m.GetId()
 	return m.(*SProxySetting), input, nil
 }

pkg/cloudcommon/db/rbac.go

@@ -70,7 +70,7 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	if !requireScope.HigherThan(scope) {
 		return nil
 	}
-	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privillege(require:%s,allow:%s)", requireScope, scope))
+	return httperrors.NewForbiddenError(fmt.Sprintf("not enough privilege(require:%s,allow:%s)", requireScope, scope))
 }
 
 func isJointObjectRbacAllowed(item IJointModel, userCred mcclient.TokenCredential, action string, extra ...string) error {

pkg/cloudcommon/db/tenantcache.go

@@ -66,8 +66,8 @@ func RegistUserCredCacheUpdater() {
 	auth.RegisterAuthHook(onAuthCompleteUpdateCache)
 }
 
-func onAuthCompleteUpdateCache(userCred mcclient.TokenCredential) {
-	TenantCacheManager.updateTenantCache(userCred)
+func onAuthCompleteUpdateCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	TenantCacheManager.updateTenantCache(ctx, userCred)
 	UserCacheManager.updateUserCache(userCred)
 }
 
@@ -91,8 +91,8 @@ func (manager *STenantCacheManager) InitializeData() error {
 	return nil
 }
 
-func (manager *STenantCacheManager) updateTenantCache(userCred mcclient.TokenCredential) {
-	manager.Save(context.Background(), userCred.GetProjectId(), userCred.GetProjectName(),
+func (manager *STenantCacheManager) updateTenantCache(ctx context.Context, userCred mcclient.TokenCredential) {
+	manager.Save(ctx, userCred.GetProjectId(), userCred.GetProjectName(),
 		userCred.GetProjectDomainId(), userCred.GetProjectDomain())
 }
 
@@ -348,14 +348,14 @@ func (manager *STenantCacheManager) findFirstProjectOfDomain(domainId string) (*
 	return &tenant, nil
 }
 
-func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId string) error {
+func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.Context, domainId string) error {
 	if len(domainId) == 0 {
 		log.Debugf("fetch empty domain!!!!")
 		debug.PrintStack()
 		return fmt.Errorf("Empty domainId")
 	}
 
-	s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
 	params := jsonutils.Marshal(map[string]string{"domain_id": domainId})
 	tenants, err := modules.Projects.List(s, params)
 	if err != nil {
@@ -366,7 +366,7 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 		tenantName, _ := tenant.GetString("name")
 		domainId, _ := tenant.GetString("domain_id")
 		domainName, _ := tenant.GetString("project_domain")
-		_, err = manager.Save(context.Background(), tenantId, tenantName, domainId, domainName)
+		_, err = manager.Save(ctx, tenantId, tenantName, domainId, domainName)
 		if err != nil {
 			return err
 		}
@@ -374,11 +374,11 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(domainId stri
 	return nil
 }
 
-func (manager *STenantCacheManager) FindFirstProjectOfDomain(domainId string) (*STenant, error) {
+func (manager *STenantCacheManager) FindFirstProjectOfDomain(ctx context.Context, domainId string) (*STenant, error) {
 	tenant, err := manager.findFirstProjectOfDomain(domainId)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			err = manager.fetchDomainTenantsFromKeystone(domainId)
+			err = manager.fetchDomainTenantsFromKeystone(ctx, domainId)
 			if err != nil {
 				return nil, errors.Wrap(err, "fetchDomainTenantsFromKeystone")
 			}

pkg/cloudcommon/policy/policy.go

@@ -80,35 +80,42 @@ type SPolicyManager struct {
 	lock *sync.Mutex
 }
 
+type sPolicyData struct {
+	Type     string               `json:"type"`
+	Enabled  bool                 `json:"enabled"`
+	DomainId string               `json:"domain_id"`
+	IsPublic bool                 `json:"is_public"`
+	Policy   jsonutils.JSONObject `json:"policy"`
+}
+
 func parseJsonPolicy(obj jsonutils.JSONObject) (string, *rbacutils.SRbacPolicy, error) {
-	typeStr, err := obj.GetString("type")
+	pData := sPolicyData{}
+	err := obj.Unmarshal(&pData)
 	if err != nil {
-		return "", nil, errors.Wrap(err, "missing type")
+		return "", nil, errors.Wrap(err, "Unmarshal")
 	}
-	domainId, err := obj.GetString("domain_id")
-	if err != nil {
-		return "", nil, errors.Wrap(err, "missing domain_id")
+	if !pData.Enabled {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "not enabled")
+	}
+	if len(pData.Type) == 0 {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing type")
 	}
 
-	isPublic := jsonutils.QueryBoolean(obj, "is_public", false)
-
-	blob, err := obj.Get("policy")
-	if err != nil {
-		log.Errorf("get blob error %s", err)
-		return "", nil, errors.Wrap(err, "json.Get")
+	if pData.Policy == nil {
+		return "", nil, errors.Wrap(httperrors.ErrInputParameter, "missing policy")
 	}
 
 	policy := rbacutils.SRbacPolicy{}
-	err = policy.Decode(blob)
+	err = policy.Decode(pData.Policy)
 	if err != nil {
 		log.Errorf("policy decode error %s", err)
 		return "", nil, errors.Wrap(err, "policy.Decode")
 	}
 
-	policy.DomainId = domainId
-	policy.IsPublic = isPublic
+	policy.DomainId = pData.DomainId
+	policy.IsPublic = pData.IsPublic
 
-	return typeStr, &policy, nil
+	return pData.Type, &policy, nil
 }
 
 func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbacPolicy, error) {
@@ -121,7 +128,7 @@ func remotePolicyFetcher() (map[rbacutils.TRbacScope]map[string]*rbacutils.SRbac
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewInt(2048), "limit")
 		params.Add(jsonutils.NewInt(int64(offset)), "offset")
-		params.Add(jsonutils.JSONTrue, "admin")
+		params.Add(jsonutils.NewString("system"), "scope")
 		params.Add(jsonutils.JSONTrue, "enabled")
 		result, err := modules.Policies.List(s, params)
 		if err != nil {
@@ -315,42 +322,16 @@ func (manager *SPolicyManager) findPolicyByName(scope rbacutils.TRbacScope, name
 }
 
 func getMatchedPolicyNames(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity) []string {
-	matchNames := make([]string, 0)
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchNames = matchNames[:0]
-		}
-		matchNames = append(matchNames, k)
-	}
+	_, matchNames := rbacutils.GetMatchedPolicies(policies, userCred)
 	return matchNames
 }
 
 func getMatchedPolicyRules(policies map[string]*rbacutils.SRbacPolicy, userCred rbacutils.IRbacIdentity, service string, resource string, action string, extra ...string) ([]rbacutils.SRbacRule, bool) {
-	matchRules := make([]rbacutils.SRbacRule, 0)
-	findMatchPolicy := false
-	maxMatchWeight := 0
-	for k := range policies {
-		isMatched, matchWeight := policies[k].Match(userCred)
-		if !isMatched || matchWeight < maxMatchWeight {
-			continue
-		}
-		if maxMatchWeight < matchWeight {
-			maxMatchWeight = matchWeight
-			matchRules = matchRules[:0]
-		}
-		findMatchPolicy = true
-		rule := policies[k].GetMatchRule(service, resource, action, extra...)
-		if rule != nil {
-			matchRules = append(matchRules, *rule)
-		}
+	matchPolicies, _ := rbacutils.GetMatchedPolicies(policies, userCred)
+	if len(matchPolicies) == 0 {
+		return nil, false
 	}
-	return matchRules, findMatchPolicy
+	return matchPolicies.GetMatchRules(service, resource, action, extra...), true
 }
 
 func (manager *SPolicyManager) allowWithoutCache(scope rbacutils.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult {
@@ -520,7 +501,7 @@ func (manager *SPolicyManager) IsScopeCapable(userCred mcclient.TokenCredential,
 	return false
 }
 
-func (manager *SPolicyManager) MatchedPolicies(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
+func (manager *SPolicyManager) MatchedPolicyNames(scope rbacutils.TRbacScope, userCred rbacutils.IRbacIdentity) []string {
 	ret := make([]string, 0)
 	policies, ok := manager.policies[scope]
 	if !ok {
@@ -544,13 +525,28 @@ func (manager *SPolicyManager) AllPolicies() map[string][]string {
 }
 
 func (manager *SPolicyManager) RoleMatchPolicies(roleName string) []string {
+	ident := rbacutils.NewRbacIdentity("", "", []string{roleName})
 	ret := make([]string, 0)
 	for _, policies := range manager.policies {
 		for name, policy := range policies {
-			if policy.MatchRole(roleName) {
+			if matched, _ := policy.Match(ident); matched {
 				ret = append(ret, name)
 			}
 		}
 	}
 	return ret
 }
+
+func (manager *SPolicyManager) GetMatchedPolicySet(userCred rbacutils.IRbacIdentity) (rbacutils.TRbacScope, rbacutils.TPolicySet) {
+	for _, scope := range []rbacutils.TRbacScope{
+		rbacutils.ScopeSystem,
+		rbacutils.ScopeDomain,
+		rbacutils.ScopeProject,
+	} {
+		macthed, _ := rbacutils.GetMatchedPolicies(manager.policies[scope], userCred)
+		if len(macthed) > 0 {
+			return scope, macthed
+		}
+	}
+	return rbacutils.ScopeNone, nil
+}

pkg/cloudcommon/policy/resources.go

@@ -66,7 +66,9 @@ var (
 	meterDomainResources = []string{}
 	meterUserResources   = []string{}
 
-	k8sSystemResources = []string{}
+	k8sSystemResources = []string{
+		"repos",
+	}
 	k8sDomainResources = []string{}
 	k8sUserResources   = []string{}
 

pkg/cloudevent/models/cloudevents.go

@@ -86,19 +86,6 @@ func (manager *SCloudeventManager) ListItemFilter(ctx context.Context, q *sqlche
 		return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
 	}
 
-	/* input.Cloudprovider = input.CloudproviderStr()
-	if len(input.Cloudprovider) > 0 {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(userCred, input.Cloudprovider)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), input.Cloudprovider)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		q = q.Equals("cloudprovider_id", providerObj.GetId())
-	}*/
-
 	if len(input.Providers) > 0 {
 		q = q.In("provider", input.Providers)
 	}
@@ -107,6 +94,22 @@ func (manager *SCloudeventManager) ListItemFilter(ctx context.Context, q *sqlche
 		q = q.In("brand", input.Brands)
 	}
 
+	if len(input.Service) > 0 {
+		q = q.In("service", input.Service)
+	}
+
+	if len(input.Manager) > 0 {
+		q = q.In("manager", input.Manager)
+	}
+
+	if len(input.Account) > 0 {
+		q = q.In("account", input.Account)
+	}
+
+	if len(input.Action) > 0 {
+		q = q.In("action", input.Action)
+	}
+
 	if !input.Since.IsZero() {
 		q = q.GT("created_at", input.Since)
 	}

pkg/cloudevent/models/cloudproviders.go

@@ -36,8 +36,10 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudevent/options"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/s3gateway/session"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 type SCloudproviderManager struct {
@@ -61,39 +63,27 @@ func init() {
 type SCloudprovider struct {
 	db.SEnabledStatusStandaloneResourceBase
 
-	HealthStatus   string `width:"16" charset:"ascii" default:"normal" nullable:"false" list:"domain"`
 	SyncStatus     string
 	LastSync       time.Time
 	LastSyncEndAt  time.Time
 	LastSyncTimeAt time.Time
 
-	AccessUrl string `width:"64" charset:"ascii" nullable:"true" list:"domain" update:"domain"`
-	Account   string `width:"128" charset:"ascii" nullable:"false" list:"domain"`
-	Secret    string `length:"0" charset:"ascii" nullable:"false" list:"domain"`
-
 	Provider string `width:"64" charset:"ascii" list:"domain"`
 	Brand    string `width:"64" charset:"ascii" list:"domain"`
-
-	ProxySetting *proxyapi.SProxySetting
 }
 
 func (manager *SCloudproviderManager) GetRegionCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) ([]SCloudprovider, error) {
 	s := session.GetSession(ctx, userCred)
 	params := jsonutils.NewDict()
-	params.Add(jsonutils.JSONTrue, "details")
 	result, err := modules.Cloudproviders.List(s, params)
 	if err != nil {
 		return nil, errors.Wrap(err, "modules.Cloudproviders.List")
 	}
+
 	providers := []SCloudprovider{}
-	for _, _provider := range result.Data {
-		provider := SCloudprovider{}
-		provider.SetModelManager(manager, &provider)
-		err = _provider.Unmarshal(&provider)
-		if err != nil {
-			return nil, errors.Wrap(err, "_provider.Unmarshal")
-		}
-		providers = append(providers, provider)
+	err = jsonutils.Update(&providers, result.Data)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.Update")
 	}
 	return providers, nil
 }
@@ -108,17 +98,18 @@ func (manager *SCloudproviderManager) GetLocalCloudproviders() ([]SCloudprovider
 	return dbProviders, nil
 }
 
-func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+func (manager *SCloudproviderManager) syncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
+	result := compare.SyncResult{}
 	providers, err := manager.GetRegionCloudproviders(ctx, userCred)
 	if err != nil {
-		log.Errorf("failed to get region cloudproviders: %v", err)
-		return
+		result.Error(errors.Wrap(err, "GetRegionCloudproviders"))
+		return result
 	}
 
 	dbProviders, err := manager.GetLocalCloudproviders()
 	if err != nil {
-		log.Errorf("failed to get local cloudproviders: %v", err)
-		return
+		result.Error(errors.Wrap(err, "GetLocalCloudproviders"))
+		return result
 	}
 
 	removed := make([]SCloudprovider, 0)
@@ -128,36 +119,48 @@ func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, us
 
 	err = compare.CompareSets(dbProviders, providers, &removed, &commondb, &commonext, &added)
 	if err != nil {
-		log.Errorf("compare.CompareSets: %v", err)
-		return
+		result.Error(errors.Wrap(err, "CompareSets"))
+		return result
 	}
 
 	for i := 0; i < len(removed); i++ {
 		err = removed[i].Delete(ctx, userCred)
 		if err != nil {
-			log.Errorf("failed to remove cloudprovider %s(%s) error: %v", removed[i].Name, removed[i].Id, err)
+			result.DeleteError(err)
+			continue
 		}
+		result.Delete()
 	}
 
 	for i := 0; i < len(commondb); i++ {
 		err = commondb[i].syncWithRegionProvider(ctx, userCred, commonext[i])
 		if err != nil {
-			log.Errorf("failed to sync cloudprovider %s(%s) error: %v", commondb[i].Name, commondb[i].Id, err)
+			result.UpdateError(err)
+			continue
 		}
+		result.Update()
 	}
 
 	for i := 0; i < len(added); i++ {
 		err = manager.newFromRegionProvider(ctx, userCred, added[i])
 		if err != nil {
-			log.Errorf("failed to add cloudprovider %s(%s) error: %v", added[i].Name, added[i].Id, err)
+			result.AddError(err)
+			continue
 		}
+		result.Add()
 	}
+	return result
+}
+
+func (manager *SCloudproviderManager) SyncCloudproviders(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	result := manager.syncCloudproviders(ctx, userCred)
+	info := result.Result()
+	log.Infof("sync cloudproviders result: %s", info)
 }
 
 func (provider *SCloudprovider) syncWithRegionProvider(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider SCloudprovider) error {
 	_, err := db.Update(provider, func() error {
 		provider.Status = cloudprovider.Status
-		provider.Secret = cloudprovider.Secret
 		provider.Enabled = cloudprovider.Enabled
 		provider.Brand = cloudprovider.Brand
 		return nil
@@ -173,8 +176,7 @@ func (self *SCloudprovider) MarkSyncing(userCred mcclient.TokenCredential) error
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to MarkSyncing error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -186,8 +188,7 @@ func (self *SCloudprovider) MarkEndSync(userCred mcclient.TokenCredential) error
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to markEndSync error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -198,8 +199,7 @@ func (self *SCloudprovider) SetLastSyncTimeAt(userCred mcclient.TokenCredential,
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to SetLastSyncTimeAt error: %v", err)
-		return err
+		return errors.Wrap(err, "db.Update")
 	}
 	return nil
 }
@@ -209,30 +209,36 @@ func (manager *SCloudproviderManager) newFromRegionProvider(ctx context.Context,
 	return manager.TableSpec().Insert(&cloudprovider)
 }
 
-func (manager *SCloudproviderManager) SyncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+func (manager *SCloudproviderManager) syncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential) error {
 	cloudproviders := []SCloudprovider{}
 	q := manager.Query().IsTrue("enabled").Equals("status", api.CLOUD_PROVIDER_CONNECTED).Equals("sync_status", api.CLOUD_PROVIDER_SYNC_STATUS_IDLE)
 	err := db.FetchModelObjects(manager, q, &cloudproviders)
 	if err != nil {
-		log.Errorf("failed to fetch cloudproviders")
-		return
+		return errors.Wrap(err, "db.FetchModelObjects")
 	}
-	for _, provider := range cloudproviders {
-		err = provider.StartCloudeventSyncTask(ctx, userCred)
+	for i := range cloudproviders {
+		err = cloudproviders[i].StartCloudeventSyncTask(ctx, userCred)
 		if err != nil {
-			log.Errorf("Failed start cloudevent sync task error: %v", err)
+			log.Errorf("Failed start cloudevent sync task for cloudprovider %s (%s) error: %v", cloudproviders[i].Name, cloudproviders[i].Id, err)
 		}
 	}
-	return
+	return nil
+}
+
+func (manager *SCloudproviderManager) SyncCloudeventTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	err := manager.syncCloudeventTask(ctx, userCred)
+	if err != nil {
+		log.Errorf("syncCloudeventTask error: %v", err)
+	}
 }
 
 func (provider *SCloudprovider) StartCloudeventSyncTask(ctx context.Context, userCred mcclient.TokenCredential) error {
 	params := jsonutils.NewDict()
-	provider.MarkSyncing(userCred)
 	task, err := taskman.TaskManager.NewTask(ctx, "CloudeventSyncTask", provider, userCred, params, "", "", nil)
 	if err != nil {
 		return errors.Wrap(err, "NewTask")
 	}
+	provider.MarkSyncing(userCred)
 	task.ScheduleRun(nil)
 	return nil
 }
@@ -277,11 +283,43 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	return start, end, nil
 }
 
-func (provider *SCloudprovider) getPassword() (string, error) {
+type SCloudproviderDelegate struct {
+	Id   string
+	Name string
+
+	Enabled    bool
+	Status     string
+	SyncStatus string
+
+	AccessUrl string
+	Account   string
+	Secret    string
+
+	Provider string
+	Brand    string
+
+	ProxySetting proxyapi.SProxySetting
+}
+
+func (self *SCloudprovider) GetDelegate() (*SCloudproviderDelegate, error) {
+	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	result, err := modules.Cloudproviders.Get(s, self.Id, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "modules.Cloudproviders.Get")
+	}
+	provider := &SCloudproviderDelegate{}
+	err = result.Unmarshal(provider)
+	if err != nil {
+		return nil, errors.Wrap(err, "result.Unmarshal")
+	}
+	return provider, nil
+}
+
+func (provider *SCloudproviderDelegate) getPassword() (string, error) {
 	return utils.DescryptAESBase64(provider.Id, provider.Secret)
 }
 
-func (provider *SCloudprovider) getAccessUrl() string {
+func (provider *SCloudproviderDelegate) getAccessUrl() string {
 	return provider.AccessUrl
 }
 
@@ -297,33 +335,40 @@ func (provider SCloudprovider) GetExternalId() string {
 	return provider.Id
 }
 
-func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) {
-	if !provider.Enabled {
+func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) {
+	if !self.Enabled {
 		return nil, errors.Error("Cloud provider is not enabled")
 	}
-	accessUrl := provider.getAccessUrl()
-	passwd, err := provider.getPassword()
+	delegate, err := self.GetDelegate()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "GetDelegate")
+	}
+	accessUrl := delegate.getAccessUrl()
+	passwd, err := delegate.getPassword()
+	if err != nil {
+		return nil, errors.Wrap(err, "delegate.getPassword")
 	}
 
-	ps := provider.ProxySetting
-	cfg := &httpproxy.Config{
-		HTTPProxy:  ps.HTTPProxy,
-		HTTPSProxy: ps.HTTPSProxy,
-		NoProxy:    ps.NoProxy,
-	}
-	cfgProxyFunc := cfg.ProxyFunc()
-	proxyFunc := func(req *http.Request) (*url.URL, error) {
-		return cfgProxyFunc(req.URL)
+	var proxyFunc httputils.TransportProxyFunc
+	{
+		cfg := &httpproxy.Config{
+			HTTPProxy:  delegate.ProxySetting.HTTPProxy,
+			HTTPSProxy: delegate.ProxySetting.HTTPSProxy,
+			NoProxy:    delegate.ProxySetting.NoProxy,
+		}
+		cfgProxyFunc := cfg.ProxyFunc()
+		proxyFunc = func(req *http.Request) (*url.URL, error) {
+			return cfgProxyFunc(req.URL)
+		}
 	}
+
 	return cloudprovider.GetProvider(
 		cloudprovider.ProviderConfig{
-			Id:      provider.Id,
-			Name:    provider.Name,
-			Vendor:  provider.Provider,
+			Id:      self.Id,
+			Name:    self.Name,
+			Vendor:  self.Provider,
 			URL:     accessUrl,
-			Account: provider.Account,
+			Account: delegate.Account,
 			Secret:  passwd,
 
 			ProxyFunc: proxyFunc,

pkg/cloudprovider/cloudprovider.go

@@ -17,6 +17,8 @@ package cloudprovider
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -33,66 +35,66 @@ const (
 )
 
 type SCloudaccountCredential struct {
-	// 账号所在的项目
+	// 账号所在的项目 (openstack)
 	ProjectName string `json:"project_name"`
 
-	// 账号所在的域
+	// 账号所在的域 (openstack)
 	// default: Default
 	DomainName string `json:"domain_name"`
 
-	// 用户名
+	// 用户名 (openstack, zstack, esxi)
 	Username string `json:"username"`
 
-	// 密码
+	// 密码 (openstack, zstack, esxi)
 	Password string `json:"password"`
 
-	// 认证地址
+	// 认证地址 (openstack,zstack)
 	AuthUrl string `json:"auto_url"`
 
-	// 秘钥id
+	// 秘钥id (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
 	AccessKeyId string `json:"access_key_id"`
 
-	// 秘钥key
+	// 秘钥key (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
 	AccessKeySecret string `json:"access_key_secret"`
 
-	// 环境
+	// 环境 (Azure, Aws, huawei, ctyun)
 	Environment string `json:"environment"`
 
-	// 目录ID
+	// 目录ID (Azure)
 	DirectoryId string `json:"directory_id"`
 
-	// 客户端ID
+	// 客户端ID (Azure)
 	ClientId string `json:"client_id"`
 
-	// 客户端秘钥
+	// 客户端秘钥 (Azure)
 	ClientSecret string `json:"client_secret"`
 
-	// 主机IP
+	// 主机IP (esxi)
 	Host string `json:"host"`
 
-	// 主机端口
+	// 主机端口 (esxi)
 	Port int `json:"port"`
 
-	// 端点
+	// 端点 (s3)
 	Endpoint string `json:"endpoint"`
 
-	// app id
+	// app id (Qcloud)
 	AppId string `json:"app_id"`
 
-	//秘钥ID
+	//秘钥ID (Qcloud)
 	SecretId string `json:"secret_id"`
 
-	//秘钥key
+	//秘钥key (Qcloud)
 	SecretKey string `json:"secret_key"`
 
-	// Google服务账号email
-	ClientEmail string `json:"client_email"`
-	// Google服务账号project id
-	ProjectId string `json:"project_id"`
-	// Google服务账号秘钥id
-	PrivateKeyId string `json:"private_key_id"`
-	// Google服务账号秘钥
-	PrivateKey string `json:"private_key"`
+	// Google服务账号email (gcp)
+	GCPClientEmail string `json:"gcp_client_email"`
+	// Google服务账号project id (gcp)
+	GCPProjectId string `json:"gcp_project_id"`
+	// Google服务账号秘钥id (gcp)
+	GCPPrivateKeyId string `json:"gcp_private_key_id"`
+	// Google服务账号秘钥 (gcp)
+	GCPPrivateKey string `json:"gcp_private_key"`
 }
 
 type SCloudaccount struct {
@@ -155,6 +157,18 @@ type ProviderConfig struct {
 	ProxyFunc httputils.TransportProxyFunc
 }
 
+func (cp *ProviderConfig) HttpClient() *http.Client {
+	client := httputils.GetClient(true, 15*time.Second)
+	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
+	return client
+}
+
+func (cp *ProviderConfig) AdaptiveTimeoutHttpClient() *http.Client {
+	client := httputils.GetAdaptiveTimeoutClient()
+	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
+	return client
+}
+
 type ICloudProviderFactory interface {
 	GetProvider(cfg ProviderConfig) (ICloudProvider, error)
 
@@ -243,7 +257,7 @@ func GetProviderFactory(provider string) (ICloudProviderFactory, error) {
 	if ok {
 		return factory, nil
 	}
-	log.Errorf("Provider %s not registerd", provider)
+	log.Errorf("Provider %s not registered", provider)
 	return nil, fmt.Errorf("No such provider %s", provider)
 }
 

pkg/cloudprovider/instance.go

@@ -37,6 +37,8 @@ const (
 	CLOUD_SHELL                 = "cloud-shell"
 	CLOUD_SHELL_WITHOUT_ENCRYPT = "cloud-shell-without-encrypt"
 	CLOUD_CONFIG                = "cloud-config"
+	CLOUD_POWER_SHELL           = "powershell"
+	CLOUD_EC2                   = "ec2"
 )
 
 type SManagedVMCreateConfig struct {

pkg/cloudprovider/objectstore.go

@@ -270,6 +270,7 @@ func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) {
 	if objs.IsTruncated {
 		return stats, errors.Wrap(httperrors.ErrTooLarge, "too many objects")
 	}
+	stats.ObjectCount, stats.SizeBytes = 0, 0
 	for _, obj := range objs.Objects {
 		stats.SizeBytes += obj.GetSizeBytes()
 		stats.ObjectCount += 1

pkg/cloudprovider/resources.go

@@ -750,7 +750,9 @@ type ICloudDBInstance interface {
 
 	GetConnectionStr() string
 	GetInternalConnectionStr() string
-	GetIZoneId() string
+	GetZone1Id() string
+	GetZone2Id() string
+	GetZone3Id() string
 	GetIVpcId() string
 
 	GetDBNetwork() (*SDBInstanceNetwork, error)

pkg/compute/baremetal/diskconfig.go

@@ -549,7 +549,7 @@ func GetDiskSpecV2(storages []*BaremetalStorage) api.DiskDriverSpec {
 		if len(driverStorages) == 0 {
 			continue
 		}
-		spec[driver] = getSpec(storages)
+		spec[driver] = getSpec(driverStorages)
 	}
 	return spec
 }

pkg/compute/guestdrivers/aws.go

@@ -78,6 +78,14 @@ func (self *SAwsGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovider
 	return true
 }
 
+func (self *SAwsGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
+func (self *SAwsGuestDriver) GetWindowsUserDataType() string {
+	return cloudprovider.CLOUD_EC2
+}
+
 func (self *SAwsGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	// return fetchAwsUserName(desc)
 	if desc.OsType == "Windows" {

pkg/compute/guestdrivers/base.go

@@ -298,6 +298,14 @@ func (self *SBaseGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovide
 	return false
 }
 
+func (self *SBaseGuestDriver) GetWindowsUserDataType() string {
+	return cloudprovider.CLOUD_POWER_SHELL
+}
+
+func (self *SBaseGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return false
+}
+
 func (self *SBaseGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_CONFIG
 }

pkg/compute/guestdrivers/managedvirtual.go

@@ -16,6 +16,7 @@ package guestdrivers
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"math"
 	"strings"
@@ -346,7 +347,15 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context
 			desc.UserData = oUserData.UserDataBase64()
 		}
 		if strings.ToLower(desc.OsType) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) {
-			desc.UserData = oUserData.UserDataPowerShell()
+			switch guest.GetDriver().GetWindowsUserDataType() {
+			case cloudprovider.CLOUD_EC2:
+				desc.UserData = oUserData.UserDataEc2()
+			default:
+				desc.UserData = oUserData.UserDataPowerShell()
+			}
+			if guest.GetDriver().IsWindowsUserDataTypeNeedEncode() {
+				desc.UserData = base64.StdEncoding.EncodeToString([]byte(desc.UserData))
+			}
 		}
 	}
 
@@ -462,20 +471,20 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte
 		return nil, err
 	}
 
+	ret, expect := 0, len(desc.DataDisks)+1
 	err = cloudprovider.RetryUntil(func() (bool, error) {
 		idisks, err := iVM.GetIDisks()
 		if err != nil {
-			log.Errorf("fail to find vm disks %s after create", err)
-			return false, err
+			return false, errors.Wrap(err, "iVM.GetIDisks")
 		}
-		if len(idisks) == len(desc.DataDisks)+1 {
+		ret = len(idisks)
+		if ret >= expect { // 有可能自定义镜像里面也有磁盘，会导致返回的磁盘多于创建时的磁盘
 			return true, nil
-		} else {
-			return false, nil
 		}
+		return false, nil
 	}, 10)
 	if err != nil {
-		return nil, errors.Wrap(err, "GuestDriver.RemoteDeployGuestForCreate.RetryUntil")
+		return nil, errors.Wrapf(err, "GuestDriver.RemoteDeployGuestForCreate.RetryUntil expect %d disks return %d disks", expect, ret)
 	}
 
 	guest.GetDriver().RemoteActionAfterGuestCreated(ctx, userCred, guest, host, iVM, &desc)
@@ -1033,7 +1042,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestAssociateEip(ctx context.Cont
 			return nil, fmt.Errorf("ManagedVirtualizedGuestDriver.RequestAssociateEip fail to local associate EIP %s", err)
 		}
 
-		eip.SetStatus(userCred, api.EIP_STATUS_READY, "associate")
+		eip.SetStatus(userCred, api.EIP_STATUS_READY, api.EIP_STATUS_ASSOCIATE)
 		return nil, nil
 	})
 

pkg/compute/guestdrivers/zstack.go

@@ -154,6 +154,10 @@ func (self *SZStackGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }
 
+func (self *SZStackGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
 func (self *SZStackGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	userName := "root"
 	if desc.OsType == "Windows" {

pkg/compute/models/buckets.go

@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	identity_apis "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -535,7 +536,7 @@ func joinPath(ep, path string) string {
 func (bucket *SBucket) getMoreDetails(out api.BucketDetails) api.BucketDetails {
 	out.CloudproviderInfo = bucket.getCloudProviderInfo()
 
-	s3gwUrl, _ := auth.GetServiceURL("s3gateway", options.Options.Region, "", "public")
+	s3gwUrl, _ := auth.GetServiceURL("s3gateway", options.Options.Region, "", identity_apis.EndpointInterfacePublic)
 	if len(s3gwUrl) > 0 {
 		accessUrls := make([]cloudprovider.SBucketAccessUrl, 0)
 		if bucket.AccessUrls != nil {
@@ -654,10 +655,10 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	ret := jsonutils.NewDict()
 	ret.Add(retArray, "data")
+	ret.Add(jsonutils.NewString("key"), "marker_field")
+	ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	if len(nextMarker) > 0 {
 		ret.Add(jsonutils.NewString(nextMarker), "next_marker")
-		ret.Add(jsonutils.NewString("key"), "marker_field")
-		ret.Add(jsonutils.NewString("DESC"), "marker_order")
 	}
 	return ret, nil
 }

pkg/compute/models/cloudaccounts.go

@@ -323,10 +323,7 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl
 	if err := v.Validate(data); err != nil {
 		return nil, err
 	}
-
-	proxySetting := v.Model.(*proxy.SProxySetting)
-
-	if proxySetting != nil {
+	if proxySetting, ok := v.Model.(*proxy.SProxySetting); ok && proxySetting.Id != self.ProxySettingId {
 		// updated proxy setting, so do the check
 		proxyFunc := proxySetting.HttpTransportProxyFunc()
 		secret, _ := self.getPassword()
@@ -401,8 +398,8 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 
 	var proxyFunc httputils.TransportProxyFunc
 	{
-		if input.ProxySettingId == "" {
-			input.ProxySettingId = proxyapi.ProxySettingId_DIRECT
+		if input.ProxySetting == "" {
+			input.ProxySetting = proxyapi.ProxySettingId_DIRECT
 		}
 		var proxySetting *proxy.SProxySetting
 		proxySetting, input.ProxySettingResourceInput, err = proxy.ValidateProxySettingResourceInput(userCred, input.ProxySettingResourceInput)
@@ -641,13 +638,13 @@ func (self *SCloudaccount) StartSyncCloudProviderInfoTask(ctx context.Context, u
 		log.Errorf("CloudAccountSyncInfoTask newTask error %s", err)
 		return err
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
 }
 
-func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -659,7 +656,7 @@ func (self *SCloudaccount) markStartSync(userCred mcclient.TokenCredential) erro
 	providers := self.GetCloudproviders()
 	for i := range providers {
 		if providers[i].Enabled {
-			err := providers[i].markStartingSync(userCred)
+			err := providers[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "providers.markStartSync")
 			}
@@ -837,7 +834,7 @@ func (self *SCloudaccount) importSubAccount(ctx context.Context, userCred mcclie
 				ownerId = userCred
 			} else {
 				// find default project of domain
-				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ownerId.GetProjectDomainId())
+				t, err := db.TenantCacheManager.FindFirstProjectOfDomain(ctx, ownerId.GetProjectDomainId())
 				if err != nil {
 					log.Errorf("cannot find a valid porject for domain %s", ownerId.GetProjectDomainId())
 					return nil, err
@@ -918,13 +915,13 @@ func (self *SCloudaccount) GetHostCount() (int, error) {
 
 func (self *SCloudaccount) GetVpcCount() (int, error) {
 	subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", self.Id).SubQuery()
-	q := VpcManager.Query().In("manager_id", subq)
+	q := VpcManager.Query().In("manager_id", subq).IsFalse("is_emulated")
 	return q.CountWithError()
 }
 
 func (self *SCloudaccount) GetStorageCount() (int, error) {
 	subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", self.Id).SubQuery()
-	q := StorageManager.Query().In("manager_id", subq)
+	q := StorageManager.Query().In("manager_id", subq).IsFalse("is_emulated")
 	return q.CountWithError()
 }
 

pkg/compute/models/cloudproviderregions.go

@@ -26,12 +26,14 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/nopanic"
 )
 
 type SCloudproviderregionManager struct {
@@ -220,17 +222,23 @@ func (manager *SCloudproviderregionManager) FetchByIdsOrCreate(providerId string
 	return cpr
 }
 
-func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudproviderregion) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	if !self.Enabled {
 		return fmt.Errorf("Cloudprovider(%s)region(%s) disabled", self.CloudproviderId, self.CloudregionId)
 	}
-	_, err := db.Update(self, func() error {
-		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
-		return nil
-	})
-	if err != nil {
-		log.Errorf("Failed to markStartingSync error: %v", err)
-		return err
+	regionIds := []string{}
+	if syncRange != nil {
+		regionIds, _ = syncRange.GetRegionIds()
+	}
+	if syncRange == nil || len(regionIds) == 0 || utils.IsInStringArray(self.CloudregionId, regionIds) {
+		_, err := db.Update(self, func() error {
+			self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
+			return nil
+		})
+		if err != nil {
+			log.Errorf("Failed to markStartingSync error: %v", err)
+			return err
+		}
 	}
 	return nil
 }
@@ -380,10 +388,12 @@ func (self *SCloudproviderregion) getSyncTaskKey() string {
 func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredential, syncRange SSyncRange, waitChan chan bool) {
 	self.markStartSync(userCred)
 	RunSyncCloudproviderRegionTask(self.getSyncTaskKey(), func() {
-		err := self.DoSync(context.Background(), userCred, syncRange)
-		if err != nil {
-			log.Errorf("DoSync faild %v", err)
-		}
+		nopanic.Run(func() {
+			err := self.DoSync(context.Background(), userCred, syncRange)
+			if err != nil {
+				log.Errorf("DoSync faild %v", err)
+			}
+		})
 		if waitChan != nil {
 			waitChan <- true
 		}

pkg/compute/models/cloudproviders.go

@@ -243,15 +243,15 @@ func (self *SCloudprovider) GetGuestCount() (int, error) {
 }
 
 func (self *SCloudprovider) GetHostCount() (int, error) {
-	return HostManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return HostManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getVpcCount() (int, error) {
-	return VpcManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return VpcManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getStorageCount() (int, error) {
-	return StorageManager.Query().Equals("manager_id", self.Id).CountWithError()
+	return StorageManager.Query().Equals("manager_id", self.Id).IsFalse("is_emulated").CountWithError()
 }
 
 func (self *SCloudprovider) getStoragecacheCount() (int, error) {
@@ -302,56 +302,60 @@ func (self *SCloudprovider) getPassword() (string, error) {
 	return utils.DescryptAESBase64(self.Id, self.Secret)
 }
 
-func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.TokenCredential) error {
-	if len(self.ProjectId) > 0 {
-		_, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
-		if err != nil && err != sql.ErrNoRows {
-			log.Errorf("fetch existing tenant by id fail %s", err)
-			return errors.Wrap(err, "db.TenantCacheManager.FetchTenantById")
-		} else if err == nil {
-			return nil // find the project, skip sync
+func getTenant(ctx context.Context, projectId string, name string) (*db.STenant, error) {
+	if len(projectId) > 0 {
+		tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId)
+		if err != nil {
+			return nil, errors.Wrap(err, "TenantCacheManager.FetchTenantById")
 		}
+		return tenant, nil
 	}
-
-	if len(self.Name) == 0 {
-		log.Errorf("syncProject: provider name is empty???")
-		return errors.Error("cannot syncProject for empty name")
+	if len(name) == 0 {
+		return nil, errors.Error("cannot syncProject for empty name")
 	}
+	return db.TenantCacheManager.FetchTenantByName(ctx, name)
+}
 
-	tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, self.Name)
-	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("fetchTenantByIdorName error %s: %s", self.Name, err)
-		return errors.Wrap(err, "db.TenantCacheManager.FetchTenantByIdOrName")
+func createTenant(ctx context.Context, name, domainId, desc string) (string, string, error) {
+	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString(name), "generate_name")
+
+	params.Add(jsonutils.NewString(domainId), "domain_id")
+	params.Add(jsonutils.NewString(desc), "description")
+
+	resp, err := modules.Projects.Create(s, params)
+	if err != nil {
+		return "", "", errors.Wrap(err, "Projects.Create")
 	}
+	projectId, err := resp.GetString("id")
+	if err != nil {
+		return "", "", errors.Wrap(err, "resp.GetString")
+	}
+	return domainId, projectId, nil
+}
 
+func getOrCreateTenant(ctx context.Context, name, domainId, projectId, desc string) (string, string, error) {
+	tenant, err := getTenant(ctx, projectId, name)
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
+		return "", "", err
+	}
+	if err == sql.ErrNoRows || tenant.DomainId != domainId {
+		return createTenant(ctx, name, domainId, desc)
+	}
+	return tenant.DomainId, tenant.Id, nil
+}
+
+func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.TokenCredential) error {
 	account := self.GetCloudaccount()
 	if account == nil {
 		return errors.Error("no valid cloudaccount???")
 	}
 
-	var projectId, domainId string
-	if err == sql.ErrNoRows || tenant.DomainId != account.DomainId { // create one
-		s := auth.GetAdminSession(ctx, options.Options.Region, "")
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(self.Name), "generate_name")
-
-		domainId = account.DomainId
-		params.Add(jsonutils.NewString(domainId), "domain_id")
-		params.Add(jsonutils.NewString(fmt.Sprintf("auto create from cloud provider %s (%s)", self.Name, self.Id)), "description")
-
-		project, err := modules.Projects.Create(s, params)
-
-		if err != nil {
-			log.Errorf("create project fail %s", err)
-			return err
-		}
-		projectId, err = project.GetString("id")
-		if err != nil {
-			return err
-		}
-	} else {
-		domainId = tenant.DomainId
-		projectId = tenant.Id
+	desc := fmt.Sprintf("auto create from cloud provider %s (%s)", self.Name, self.Id)
+	domainId, projectId, err := getOrCreateTenant(ctx, self.Name, account.DomainId, self.ProjectId, desc)
+	if err != nil {
+		return errors.Wrap(err, "getOrCreateTenant")
 	}
 
 	return self.saveProject(userCred, domainId, projectId)
@@ -384,6 +388,44 @@ type SSyncRange struct {
 	Host   []string
 }
 
+func (sr *SSyncRange) GetRegionIds() ([]string, error) {
+	regionIds := []string{}
+	if len(sr.Host) == 0 && len(sr.Zone) == 0 && len(sr.Region) == 0 {
+		return regionIds, nil
+	}
+	hostQ := HostManager.Query().SubQuery()
+	hosts := hostQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(hostQ.Field("id"), sr.Host),
+		sqlchemy.In(hostQ.Field("name"), sr.Host),
+	)).SubQuery()
+	zoneQ := ZoneManager.Query().SubQuery()
+	zones := zoneQ.Query().Filter(sqlchemy.OR(
+		sqlchemy.In(zoneQ.Field("id"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("name"), sr.Zone),
+		sqlchemy.In(zoneQ.Field("id"), hosts.Query(hosts.Field("zone_id")).SubQuery()),
+	)).SubQuery()
+	regionQ := CloudregionManager.Query().SubQuery()
+	q := regionQ.Query(regionQ.Field("id")).Filter(sqlchemy.OR(
+		sqlchemy.In(regionQ.Field("id"), sr.Region),
+		sqlchemy.In(regionQ.Field("name"), sr.Region),
+		sqlchemy.In(regionQ.Field("id"), zones.Query(zones.Field("cloudregion_id")).SubQuery()),
+	))
+	rows, err := q.Rows()
+	if err != nil {
+		return nil, errors.Wrap(err, "q.Rows")
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var regionId string
+		err = rows.Scan(&regionId)
+		if err != nil {
+			return nil, errors.Wrap(err, "rows.Scan")
+		}
+		regionIds = append(regionIds, regionId)
+	}
+	return regionIds, nil
+}
+
 func (sr *SSyncRange) NeedSyncInfo() bool {
 	if sr.FullSync {
 		return true
@@ -539,7 +581,7 @@ func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context,
 		cloudaccount.markAutoSync(userCred)
 		cloudaccount.MarkSyncing(userCred)
 	}
-	self.markStartSync(userCred)
+	self.markStartSync(userCred, syncRange)
 	db.OpsLog.LogEvent(self, db.ACT_SYNC_HOST_START, "", userCred)
 	task.ScheduleRun(nil)
 	return nil
@@ -606,7 +648,7 @@ func (self *SCloudprovider) PerformChangeProject(ctx context.Context, userCred m
 	return nil, self.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true, DeepSync: true}, "")
 }
 
-func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUING
 		return nil
@@ -618,7 +660,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -627,7 +669,7 @@ func (self *SCloudprovider) markStartingSync(userCred mcclient.TokenCredential)
 	return nil
 }
 
-func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) error {
+func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential, syncRange *SSyncRange) error {
 	_, err := db.Update(self, func() error {
 		self.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED
 		return nil
@@ -639,7 +681,7 @@ func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) err
 	cprs := self.GetCloudproviderRegions()
 	for i := range cprs {
 		if cprs[i].Enabled {
-			err := cprs[i].markStartingSync(userCred)
+			err := cprs[i].markStartingSync(userCred, syncRange)
 			if err != nil {
 				return errors.Wrap(err, "cprs[i].markStartingSync")
 			}
@@ -1136,9 +1178,10 @@ func (provider *SCloudprovider) GetCloudproviderRegions() []SCloudproviderregion
 func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange, wg *sync.WaitGroup, autoSync bool) {
 	provider.markSyncing(userCred)
 	cprs := provider.GetCloudproviderRegions()
+	regionIds, _ := syncRange.GetRegionIds()
 	syncCnt := 0
 	for i := range cprs {
-		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) {
+		if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) && (len(regionIds) == 0 || utils.IsInStringArray(cprs[i].CloudregionId, regionIds)) {
 			syncCnt += 1
 			var waitChan chan bool = nil
 			if wg != nil {

pkg/compute/models/cloudregions.go

@@ -406,6 +406,9 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 	}
 
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
+		if !utils.IsInStringArray(self.Provider, api.PRIVATE_CLOUD_PROVIDERS) {
+			self.Name = cloudRegion.GetName()
+		}
 		self.Status = cloudRegion.GetStatus()
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()

pkg/compute/models/dbinstance_accounts.go

@@ -60,7 +60,8 @@ type SDBInstanceAccount struct {
 	db.SExternalizedResourceBase
 
 	// 数据库密码
-	Secret string `width:"256" charset:"ascii" nullable:"false" list:"domain" create:"optional"`
+	Secret string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"optional"`
+
 	// RDS实例Id
 	DBInstanceId string `width:"36" charset:"ascii" name:"dbinstance_id" nullable:"false" list:"user" create:"required" index:"true"`
 }
@@ -100,7 +101,7 @@ func (manager *SDBInstanceAccountManager) FetchOwnerId(ctx context.Context, data
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceAccountManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_databases.go

@@ -100,7 +100,7 @@ func (manager *SDBInstanceDatabaseManager) FetchOwnerId(ctx context.Context, dat
 		}
 		return instance.(*SDBInstance).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SDBInstanceDatabaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/dbinstance_skus.go

@@ -59,7 +59,7 @@ type SDBInstanceSku struct {
 	SCloudregionResourceBase
 	Provider string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin"`
 
-	StorageType   string `list:"user" create:"optional"`
+	StorageType   string `width:"32" index:"true" list:"user" create:"optional"`
 	DiskSizeStep  int    `list:"user" default:"1" create:"optional"` //步长
 	MaxDiskSizeGb int    `list:"user" create:"optional"`
 	MinDiskSizeGb int    `list:"user" create:"optional"`
@@ -72,9 +72,9 @@ type SDBInstanceSku struct {
 	VcpuCount  int `nullable:"false" default:"1" list:"user" create:"optional"`
 	VmemSizeMb int `nullable:"false" list:"user" create:"required"`
 
-	Category      string `nullable:"false" list:"user" create:"optional"`
-	Engine        string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
-	EngineVersion string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	Category      string `width:"32" index:"true" nullable:"false" list:"user" create:"optional"`
+	Engine        string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	EngineVersion string `width:"16" index:"true" charset:"ascii" nullable:"false" list:"user" create:"required"`
 
 	Zone1  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`
 	Zone2  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"`

pkg/compute/models/dbinstances.go

@@ -123,9 +123,6 @@ type SDBInstance struct {
 	Zone2 string `width:"36" charset:"ascii" nullable:"false" create:"optional" list:"user"`
 	// 可用区3
 	Zone3 string `width:"36" charset:"ascii" nullable:"false" create:"optional" list:"user"`
-
-	// 可用区Id(对应公有云的可用区Id)
-	ZoneId string `width:"36" charset:"ascii" nullable:"false" create:"optional"`
 }
 
 func (manager *SDBInstanceManager) GetContextManagers() [][]db.IModelManager {
@@ -383,14 +380,6 @@ func (self *SDBInstance) GetVpc() (*SVpc, error) {
 	return vpc.(*SVpc), nil
 }
 
-func (self *SDBInstance) GetZone() (*SZone, error) {
-	zone, err := ZoneManager.FetchById(self.ZoneId)
-	if err != nil {
-		return nil, err
-	}
-	return zone.(*SZone), nil
-}
-
 func (self *SDBInstance) GetNetwork() (*SNetwork, error) {
 	dbnet := DBInstanceNetworkManager.Query().SubQuery()
 	q := NetworkManager.Query()
@@ -1182,7 +1171,7 @@ func (self *SDBInstance) ValidateDeleteCondition(ctx context.Context) error {
 func (self *SDBInstance) GetDBInstanceSkuQuery() *sqlchemy.SQuery {
 	q := DBInstanceSkuManager.Query().Equals("storage_type", self.StorageType).Equals("category", self.Category).
 		Equals("cloudregion_id", self.CloudregionId).Equals("engine", self.Engine).Equals("engine_version", self.EngineVersion)
-	for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3, "zone_id": self.ZoneId} {
+	for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3} {
 		if len(v) > 0 {
 			q = q.Equals(k, v)
 		}
@@ -1285,6 +1274,30 @@ func (self *SDBInstance) SetZoneInfo(ctx context.Context, userCred mcclient.Toke
 	return err
 }
 
+func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) {
+	zone1 := extInstance.GetZone1Id()
+	if len(zone1) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone1)
+		if zone != nil {
+			self.Zone1 = zone.GetId()
+		}
+	}
+	zone2 := extInstance.GetZone2Id()
+	if len(zone2) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone2)
+		if zone != nil {
+			self.Zone2 = zone.GetId()
+		}
+	}
+	zone3 := extInstance.GetZone3Id()
+	if len(zone3) > 0 {
+		zone, _ := db.FetchByExternalId(ZoneManager, zone3)
+		if zone != nil {
+			self.Zone3 = zone.GetId()
+		}
+	}
+}
+
 func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extInstance cloudprovider.ICloudDBInstance) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Engine = extInstance.GetEngine()
@@ -1295,22 +1308,22 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 		self.DiskSizeGB = extInstance.GetDiskSizeGB()
 		self.StorageType = extInstance.GetStorageType()
 		self.Status = extInstance.GetStatus()
+		self.Port = extInstance.GetPort()
 
 		self.ConnectionStr = extInstance.GetConnectionStr()
 		self.InternalConnectionStr = extInstance.GetInternalConnectionStr()
 
 		self.MaintainTime = extInstance.GetMaintainTime()
-
-		self.ZoneId = extInstance.GetIZoneId()
-		err := self.setZoneInfo()
-		if err != nil {
-			log.Errorf("failed to set zone info for dbinstance %s(%s) error: %v", self.Name, self.Id, err)
-		}
+		self.SetZoneIds(extInstance)
 
 		if createdAt := extInstance.GetCreatedAt(); !createdAt.IsZero() {
 			self.CreatedAt = createdAt
 		}
 
+		if expiredAt := extInstance.GetExpiredAt(); !expiredAt.IsZero() {
+			self.ExpiredAt = expiredAt
+		}
+
 		factory, err := provider.GetProviderFactory()
 		if err != nil {
 			return errors.Wrap(err, "SyncWithCloudDBInstance.GetProviderFactory")
@@ -1368,11 +1381,7 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	instance.InternalConnectionStr = extInstance.GetInternalConnectionStr()
 
 	instance.MaintainTime = extInstance.GetMaintainTime()
-	instance.ZoneId = extInstance.GetIZoneId()
-	err = instance.setZoneInfo()
-	if err != nil {
-		log.Errorf("failed to set zone info for dbinstance %s error: %v", instance.ExternalId, err)
-	}
+	instance.SetZoneIds(extInstance)
 
 	if secgroupId := extInstance.GetSecurityGroupId(); len(secgroupId) > 0 {
 		q := SecurityGroupCacheManager.Query().Equals("manager_id", provider.Id).Equals("external_id", secgroupId)

pkg/compute/models/disks.go

@@ -830,6 +830,9 @@ func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredenti
 	}
 	addDisk := sizeMb - disk.DiskSize
 	storage := disk.GetStorage()
+	if storage == nil {
+		return httperrors.NewInternalServerError("disk has no valid storage")
+	}
 	if host := storage.GetMasterHost(); host != nil {
 		if err := host.GetHostDriver().ValidateDiskSize(storage, sizeMb>>10); err != nil {
 			return httperrors.NewInputParameterError(err.Error())
@@ -986,6 +989,10 @@ func (self *SDisk) ValidatePurgeCondition(ctx context.Context) error {
 func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) error {
 	if !isPurge {
 		storage := self.GetStorage()
+		if storage == nil {
+			// storage is empty, a dirty data, allow delete
+			return nil
+		}
 		host := storage.GetMasterHost()
 		if host == nil {
 			return httperrors.NewBadRequestError("storage of disk no valid host")
@@ -1098,6 +1105,9 @@ func (self *SDisk) GetPathAtHost(host *SHost) string {
 
 func (self *SDisk) GetFetchUrl() string {
 	storage := self.GetStorage()
+	if storage == nil {
+		return ""
+	}
 	host := storage.GetMasterHost()
 	return fmt.Sprintf("%s/disks/%s", host.GetFetchUrl(true), self.Id)
 }
@@ -1323,7 +1333,6 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	}
 	extDisk.Refresh()
 
-	storage := self.GetStorage()
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		// self.Name = extDisk.GetName()
 		self.Status = extDisk.GetStatus()
@@ -1350,8 +1359,12 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 		self.IsEmulated = extDisk.IsEmulated()
 
 		if provider.GetFactory().IsSupportPrepaidResources() && !recycle {
-			self.BillingType = extDisk.GetBillingType()
-			self.ExpiredAt = extDisk.GetExpiredAt()
+			if billintType := extDisk.GetBillingType(); len(billintType) > 0 {
+				self.BillingType = extDisk.GetBillingType()
+			}
+			if expiredAt := extDisk.GetExpiredAt(); !expiredAt.IsZero() {
+				self.ExpiredAt = extDisk.GetExpiredAt()
+			}
 		}
 
 		if createdAt := extDisk.GetCreatedAt(); !createdAt.IsZero() {
@@ -1370,6 +1383,10 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	if err != nil {
 		return errors.Wrapf(err, "Get snapshot policies of ICloudDisk %s.", extDisk.GetId())
 	}
+	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	err = SnapshotPolicyDiskManager.SyncByDisk(ctx, userCred, snapshotpolicies, syncOwnerId, self, storage)
 	if err != nil {
 		return err
@@ -1527,7 +1544,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 	// 	diskConfig.SizeMb = options.Options.DefaultDiskSize // MB
 	// else
 	if len(info.ImageId) == 0 && info.SizeMb == 0 {
-		return nil, httperrors.NewInputParameterError("Diskinfo not contains either imageID or size")
+		return nil, httperrors.NewInputParameterError("Diskinfo index %d: both imageID and size are absent", info.Index)
 	}
 	return info, nil
 }
@@ -1905,6 +1922,9 @@ func (self *SDisk) SwitchToBackup(userCred mcclient.TokenCredential) error {
 
 func (self *SDisk) ClearHostSchedCache() error {
 	storage := self.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
 	hosts := storage.GetAllAttachingHosts()
 	if hosts == nil {
 		return fmt.Errorf("get attaching host error")
@@ -2064,7 +2084,11 @@ func (disk *SDisk) validateDiskAutoCreateSnapshot() error {
 	if len(guests) == 0 {
 		return fmt.Errorf("Disks %s not attach guest, can't create snapshot", disk.GetName())
 	}
-	if len(guests) == 1 && utils.IsInStringArray(disk.GetStorage().StorageType, api.FIEL_STORAGE) {
+	storage := disk.GetStorage()
+	if storage == nil {
+		return fmt.Errorf("no valid storage")
+	}
+	if len(guests) == 1 && utils.IsInStringArray(storage.StorageType, api.FIEL_STORAGE) {
 		if !utils.IsInStringArray(guests[0].Status, []string{api.VM_RUNNING, api.VM_READY}) {
 			return fmt.Errorf("Guest(%s) in status(%s) cannot do disk snapshot", guests[0].Id, guests[0].Status)
 		}
@@ -2245,6 +2269,9 @@ func (self *SDisk) GetDynamicConditionInput() *jsonutils.JSONDict {
 
 func (self *SDisk) IsNeedWaitSnapshotsDeleted() (bool, error) {
 	storage := self.GetStorage()
+	if storage == nil {
+		return false, fmt.Errorf("no valid storage")
+	}
 	if storage.StorageType == api.STORAGE_RBD {
 		scnt, err := self.GetSnapshotCount()
 		if err != nil {
@@ -2311,7 +2338,12 @@ func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCre
 	}
 	provider := self.GetCloudprovider()
 	syncOwnerId := provider.GetOwnerId()
-	region := self.GetStorage().GetRegion()
+	storage := self.GetStorage()
+	if storage == nil {
+		syncResult.Error(fmt.Errorf("no valid storage"))
+		return syncResult
+	}
+	region := storage.GetRegion()
 
 	extSnapshots, err := extDisk.GetISnapshots()
 	if err != nil {

pkg/compute/models/dnsrecords.go

@@ -37,6 +37,8 @@ type SDnsRecordManager struct {
 	db.SAdminSharableVirtualResourceBaseManager
 }
 
+var _ db.IAdminSharableVirtualModelManager = DnsRecordManager
+
 var DnsRecordManager *SDnsRecordManager
 
 func init() {
@@ -276,35 +278,38 @@ func (man *SDnsRecordManager) checkRecordValue(typ, val string) error {
 
 func (man *SDnsRecordManager) validateModelData(
 	ctx context.Context,
-	userCred mcclient.TokenCredential,
-	ownerId mcclient.IIdentityProvider,
-	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
-) (*jsonutils.JSONDict, error) {
-	records, err := man.ParseInputInfo(data)
+	isCreate bool,
+) (records []string, err error) {
+	data.Remove("records")
+	records, err = man.ParseInputInfo(data)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if len(records) == 0 {
-		return nil, httperrors.NewInputParameterError("Empty record")
+		if isCreate {
+			err = httperrors.NewInputParameterError("Empty record")
+			return
+		}
+		return
 	}
 	recType := man.getRecordsType(records)
 	name, err := data.GetString("name")
 	if err != nil {
-		return nil, err
+		return
 	}
 	err = man.checkRecordName(recType, name)
 	if err != nil {
-		return nil, err
+		return
 	}
 	if data.Contains("ttl") {
-		jo, err := data.Get("ttl")
+		var (
+			ttl int64
+		)
+		ttl, err = data.Int("ttl")
 		if err != nil {
-			return nil, err
-		}
-		ttl, err := jo.Int()
-		if err != nil {
-			return nil, httperrors.NewInputParameterError("invalid ttl: %s", err)
+			err = httperrors.NewInputParameterError("invalid ttl: %s", err)
+			return
 		}
 		if ttl == 0 {
 			// - Create: use the database default
@@ -312,10 +317,11 @@ func (man *SDnsRecordManager) validateModelData(
 			data.Remove("ttl")
 		} else if ttl < 0 || ttl > 0x7fffffff {
 			// positive values of a signed 32 bit number.
-			return nil, httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			err = httperrors.NewInputParameterError("invalid ttl: %d", ttl)
+			return
 		}
 	}
-	return data, err
+	return records, nil
 }
 
 func (man *SDnsRecordManager) ValidateCreateData(
@@ -325,7 +331,7 @@ func (man *SDnsRecordManager) ValidateCreateData(
 	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
-	data, err := man.validateModelData(ctx, userCred, ownerId, query, data)
+	_, err := man.validateModelData(ctx, data, true)
 	if err != nil {
 		return nil, err
 	}
@@ -402,15 +408,11 @@ func (rec *SDnsRecord) GetInfo() []string {
 
 func (rec *SDnsRecord) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	data.UpdateDefault(jsonutils.Marshal(rec))
-	data, err := DnsRecordManager.validateModelData(ctx, userCred, rec.GetOwnerId(), query, data)
+	records, err := DnsRecordManager.validateModelData(ctx, data, false)
 	if err != nil {
 		return nil, err
 	}
-	{
-		records, err := DnsRecordManager.ParseInputInfo(data)
-		if err != nil {
-			return nil, err
-		}
+	if len(records) > 0 {
 		data.Set("records", jsonutils.NewString(strings.Join(records, DNS_RECORDS_SEPARATOR)))
 	}
 	return rec.SAdminSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)

pkg/compute/models/elasticcache_instances.go

@@ -116,7 +116,7 @@ func elasticcacheSubResourceFetchOwnerId(ctx context.Context, data jsonutils.JSO
 		return ec.(*SElasticcache).GetOwnerId(), nil
 	}
 
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 // elastic cache 子资源获取owner query

pkg/compute/models/external_projects.go

@@ -182,8 +182,10 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
-		self.ProjectId = provider.ProjectId
-		self.DomainId = provider.DomainId
+		if self.DomainId != provider.DomainId {
+			self.ProjectId = provider.ProjectId
+			self.DomainId = provider.DomainId
+		}
 		return nil
 	})
 	if err != nil {
@@ -198,18 +200,25 @@ func (manager *SExternalProjectManager) newFromCloudProject(ctx context.Context,
 	project := SExternalProject{}
 	project.SetModelManager(manager, &project)
 
-	newName, err := db.GenerateName(manager, userCred, extProject.GetName())
-	if err != nil {
-		return nil, err
-	}
-	project.Name = newName
+	project.Name = extProject.GetName()
 	project.ExternalId = extProject.GetGlobalId()
 	project.IsEmulated = extProject.IsEmulated()
 	project.ManagerId = provider.Id
 	project.DomainId = provider.DomainId
 	project.ProjectId = provider.ProjectId
+	account := provider.GetCloudaccount()
+	if account != nil && account.AutoCreateProject {
+		desc := fmt.Sprintf("auto create from cloud project %s (%s)", project.Name, project.ExternalId)
+		domainId, projectId, err := getOrCreateTenant(ctx, project.Name, provider.DomainId, "", desc)
+		if err != nil {
+			log.Errorf("failed to get or create tenant %s(%s) %v", project.Name, project.ExternalId, err)
+		} else {
+			project.DomainId = domainId
+			project.ProjectId = projectId
+		}
+	}
 
-	err = manager.TableSpec().Insert(&project)
+	err := manager.TableSpec().Insert(&project)
 	if err != nil {
 		log.Errorf("newFromCloudProject fail %s", err)
 		return nil, err

pkg/compute/models/guest_actions.go

@@ -3991,6 +3991,20 @@ func (self *SGuest) PerformSyncFixNics(ctx context.Context,
 		return nil, httperrors.NewInputParameterError("missing field ip, list of ip")
 	}
 	iplist := iplistArray.(*jsonutils.JSONArray).GetStringArray()
+	errs := make([]error, 0)
+	for i := range vnics {
+		ip := vnics[i].GetIP()
+		if len(ip) == 0 {
+			continue
+		}
+		_, err := host.getNetworkOfIPOnHost(ip)
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, ip))
+		}
+	}
+	if len(errs) > 0 {
+		return nil, httperrors.NewInvalidStatusError(errors.NewAggregate(errs).Error())
+	}
 	result := self.SyncVMNics(ctx, userCred, host, vnics, iplist)
 	if result.IsError() {
 		return nil, httperrors.NewInternalServerError(result.Result())

pkg/compute/models/guest_template.go

@@ -246,8 +246,8 @@ func (gt *SGuestTemplate) getMoreDetails(ctx context.Context, userCred mcclient.
 	configInfo.Disks = disks
 
 	// keypair
-	if len(input.KeypairId) > 0 {
-		model, err := KeypairManager.FetchById(input.KeypairId)
+	if len(input.Keypair) > 0 {
+		model, err := KeypairManager.FetchByIdOrName(userCred, input.Keypair)
 		if err == nil {
 			keypair := model.(*SKeypair)
 			configInfo.Keypair = keypair.GetName()
@@ -341,7 +341,7 @@ func (gt *SGuestTemplate) PerformPublic(ctx context.Context, userCred mcclient.T
 
 	// check for below private resource in the guest template
 	privateResource := map[string]int{
-		"keypair":           len(input.KeypairId),
+		"keypair":           len(input.Keypair),
 		"instance group":    len(input.InstanceGroupIds),
 		"instance snapshot": len(input.InstanceSnapshotId),
 	}

pkg/compute/models/guestdisks.go

@@ -178,15 +178,11 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 			desc.Add(jsonutils.NewString(storagecacheimg.Path), "image_path")
 		}
 	}
-	storage := disk.GetStorage()
-	// XXX ???
 	if host.HostType == api.HOST_TYPE_HYPERVISOR {
 		desc.Add(jsonutils.NewString(disk.StorageId), "storage_id")
 		localpath := disk.GetPathAtHost(host)
 		if len(localpath) == 0 {
 			desc.Add(jsonutils.JSONTrue, "migrating")
-			target := host.GetLeastUsedStorage(storage.StorageType)
-			desc.Add(jsonutils.NewString(target.Id), "target_storage_id")
 			disk.SetStatus(nil, api.DISK_START_MIGRATE, "migration")
 		} else {
 			desc.Add(jsonutils.NewString(localpath), "path")

pkg/compute/models/guestdrivers.go

@@ -178,6 +178,8 @@ type IGuestDriver interface {
 
 	IsNeedInjectPasswordByCloudInit(desc *cloudprovider.SManagedVMCreateConfig) bool
 	GetUserDataType() string
+	GetWindowsUserDataType() string
+	IsWindowsUserDataTypeNeedEncode() bool
 	CancelExpireTime(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error
 
 	IsSupportCdrom(guest *SGuest) (bool, error)

pkg/compute/models/guests.go

@@ -1116,6 +1116,9 @@ func (manager *SGuestManager) validateCreateData(
 				rootDiskConfig.SizeMb = sysMinDiskMB
 			}
 		}
+		if len(rootDiskConfig.Driver) == 0 {
+			rootDiskConfig.Driver = osProf.DiskDriver
+		}
 		log.Debugf("ROOT DISK: %#v", rootDiskConfig)
 		input.Disks[0] = rootDiskConfig
 		//data.Set("disk.0", jsonutils.Marshal(rootDiskConfig))
@@ -1211,13 +1214,13 @@ func (manager *SGuestManager) validateCreateData(
 		input.IsolatedDevices[idx] = devConfig
 	}
 
-	keypairId := input.KeypairId
+	keypairId := input.Keypair
 	if len(keypairId) > 0 {
 		keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairId)
 		if err != nil {
 			return nil, httperrors.NewResourceNotFoundError("Keypair %s not found", keypairId)
 		}
-		input.KeypairId = keypairObj.GetId()
+		input.Keypair = keypairObj.GetId()
 	}
 
 	secGrpIds := []string{}
@@ -3185,14 +3188,19 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 		return nil, err
 	}
 
-	cancelUsage := SQuota{}
-	cancelUsage.Storage = disk.DiskSize
-	keys, err := self.GetQuotaKeys()
-	if err != nil {
-		return nil, err
+	if pendingUsage != nil {
+		cancelUsage := SQuota{}
+		cancelUsage.Storage = disk.DiskSize
+		keys, err := self.GetQuotaKeys()
+		if err != nil {
+			return nil, err
+		}
+		cancelUsage.SetKeys(keys)
+		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
+		if err != nil {
+			return nil, err
+		}
 	}
-	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 
 	return disk, nil
 }
@@ -4684,7 +4692,9 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 		for idx, disk := range genInput.Disks {
 			tmpD := disk
 			if idx < len(userInput.Disks) {
-				tmpD.Schedtags = userInput.Disks[idx].Schedtags
+				inputDisk := userInput.Disks[idx]
+				tmpD.Schedtags = inputDisk.Schedtags
+				tmpD.Storage = inputDisk.Storage
 			}
 			disks = append(disks, tmpD)
 		}
@@ -4694,7 +4704,9 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 	for idx, net := range genInput.Networks {
 		tmpN := net
 		if idx < len(userInput.Networks) {
-			tmpN.Schedtags = userInput.Disks[idx].Schedtags
+			inputNet := userInput.Networks[idx]
+			tmpN.Schedtags = inputNet.Schedtags
+			tmpN.Network = inputNet.Network
 		}
 		nets = append(nets, tmpN)
 	}
@@ -4714,7 +4726,7 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 	userInput.ShutdownBehavior = genInput.ShutdownBehavior
 	userInput.IsSystem = genInput.IsSystem
 	userInput.SecgroupId = genInput.SecgroupId
-	userInput.KeypairId = genInput.KeypairId
+	userInput.Keypair = genInput.Keypair
 	userInput.EipBw = genInput.EipBw
 	userInput.EipChargeType = genInput.EipChargeType
 	// cloned server should belongs to the project creating it
@@ -4776,7 +4788,7 @@ func (self *SGuest) toCreateInput() *api.ServerCreateInput {
 	r.IsolatedDevices = self.ToIsolatedDevicesConfig()
 
 	if keypair := self.getKeypair(); keypair != nil {
-		r.KeypairId = keypair.Id
+		r.Keypair = keypair.Id
 	}
 	if host := self.GetHost(); host != nil {
 		r.ResourceType = host.ResourceType

pkg/compute/models/hosts.go

@@ -3660,10 +3660,16 @@ func (self *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCrede
 			}
 			net, err = wire.GetCandidatePrivateNetwork(userCred, false, netTypes)
 			if err != nil {
-				return fmt.Errorf("fail to find network %s", err)
+				return fmt.Errorf("fail to find private network %s", err)
 			}
 			if net == nil {
-				return fmt.Errorf("No network found")
+				net, err = wire.GetCandidatePublicNetwork(false, netTypes)
+				if err != nil {
+					return fmt.Errorf("fail to find public network %s", err)
+				}
+				if net == nil {
+					return fmt.Errorf("No network found")
+				}
 			}
 		}
 	} else if net.WireId != wire.Id {

pkg/compute/models/instance_snapshots.go

@@ -262,7 +262,7 @@ func (self *SInstanceSnapshot) ToInstanceCreateInput(
 		sourceInput.VcpuCount = serverConfig.Ncpu
 	}
 	if len(self.KeypairId) > 0 {
-		sourceInput.KeypairId = self.KeypairId
+		sourceInput.Keypair = self.KeypairId
 	}
 	if self.SecGroups != nil {
 		secGroups := make([]string, 0)

pkg/compute/models/loadbalanceragents.go

@@ -31,6 +31,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	identity_apis "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/compute/options"
@@ -247,7 +248,8 @@ func (p *SLoadbalancerAgentParamsTelegraf) Validate(data *jsonutils.JSONDict) er
 func (p *SLoadbalancerAgentParamsTelegraf) initDefault(data *jsonutils.JSONDict) {
 	if p.InfluxDbOutputUrl == "" {
 		baseOpts := &options.Options
-		u, _ := auth.GetServiceURL("influxdb", baseOpts.Region, "", "")
+		u, _ := auth.GetServiceURL("influxdb", baseOpts.Region, "",
+			identity_apis.EndpointInterfacePublic)
 		p.InfluxDbOutputUrl = u
 	}
 	if p.HaproxyInputInterval == 0 {
@@ -440,9 +442,11 @@ func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx contex
 	}
 	agentsData := jsonutils.Marshal(&agents).(*jsonutils.JSONArray)
 	for fieldName, man := range men {
-		keyPlural := man.KeywordPlural()
-		now := time.Now()
-		minT := now
+		var (
+			keyPlural = man.KeywordPlural()
+			now       = time.Now()
+			minT      = now
+		)
 		if len(agents) > 0 {
 			// find min updated_at seen by these active agents
 			for i := 0; i < agentsData.Length(); i++ {
@@ -471,17 +475,13 @@ func (man *SLoadbalancerAgentManager) CleanPendingDeleteLoadbalancers(ctx contex
 				continue
 			}
 			defer rows.Close()
-			m, err := db.NewModelObject(man)
-			if err != nil {
-				log.Errorf("%s: new model object failed: %s", keyPlural, err)
-				continue
-			}
-			mInitValue := reflect.Indirect(reflect.ValueOf(m))
-			m, _ = db.NewModelObject(man)
 			for rows.Next() {
-				reflect.Indirect(reflect.ValueOf(m)).Set(mInitValue)
-				err := q.Row2Struct(rows, m)
+				m, err := db.NewModelObject(man)
 				if err != nil {
+					log.Errorf("%s: new model object failed: %s", keyPlural, err)
+					continue
+				}
+				if err := q.Row2Struct(rows, m); err != nil {
 					log.Errorf("%s: Row2Struct: %s", keyPlural, err)
 					continue
 				}

pkg/compute/models/loadbalanceragents_deploy.go

@@ -216,12 +216,9 @@ func (lbagent *SLoadbalancerAgent) undeploy(ctx context.Context, userCred mcclie
 		},
 		Modules: []ansible.Module{
 			{
-				Name: "systemd",
+				Name: "shell",
 				Args: []string{
-					"name=yunion-lbagent",
-					"enabled=no",
-					"state=stopped",
-					"daemon_reload=yes",
+					"systemctl disable --now yunion-lbagent; true",
 				},
 			},
 			{

pkg/compute/models/loadbalancerawscachedlbbg.go

@@ -229,14 +229,8 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 	}
 	for i := 0; i < len(commondb); i++ {
 		var elb *SLoadbalancer
-		elbIds := commonext[i].GetLoadbalancerId()
-		if err != nil {
-			syncResult.UpdateError(err)
-			continue
-		}
-
 		elbId := commonext[i].GetLoadbalancerId()
-		if len(elbIds) > 0 {
+		if len(elbId) > 0 {
 			ielb, err := db.FetchByExternalId(LoadbalancerManager, elbId)
 			if err == nil {
 				elb = ielb.(*SLoadbalancer)
@@ -244,10 +238,8 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 		}
 
 		if elb == nil {
-			elb = &SLoadbalancer{}
-			elb.Id = ""
-			elb.CloudregionId = region.GetId()
-			elb.ManagerId = provider.GetId()
+			log.Debugf("Aws.SyncLoadbalancerBackendgroups skiped external backendgroup %s", elbId)
+			continue
 		}
 
 		err = commondb[i].SyncWithCloudLoadbalancerBackendgroup(ctx, userCred, elb, commonext[i], provider.GetOwnerId())
@@ -264,23 +256,18 @@ func (man *SAwsCachedLbbgManager) SyncLoadbalancerBackendgroups(ctx context.Cont
 	for i := 0; i < len(added); i++ {
 		var elb *SLoadbalancer
 		elbId := added[i].GetLoadbalancerId()
-		if err != nil {
-			syncResult.AddError(err)
-			continue
-		}
-
 		if len(elbId) > 0 {
 			elb, err = LoadbalancerManager.FetchByExternalId(provider.GetId(), elbId)
 			if err != nil {
 				log.Debugf("awsCachedLbbgManager.SyncLoadbalancerBackendgroups %s", err)
+				syncResult.AddError(err)
+				continue
 			}
 		}
 
 		if elb == nil {
-			elb = &SLoadbalancer{}
-			elb.Id = ""
-			elb.CloudregionId = region.GetId()
-			elb.ManagerId = provider.GetId()
+			log.Debugf("Aws.SyncLoadbalancerBackendgroups skiped external backendgroup %s", elbId)
+			continue
 		}
 
 		new, err := man.newFromCloudLoadbalancerBackendgroup(ctx, userCred, elb, added[i], syncOwnerId)

pkg/compute/models/loadbalancerbackendgroups.go

@@ -102,6 +102,15 @@ func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context,
 	return q, nil
 }
 
+func (man *SLoadbalancerBackendGroupManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", nil)
+	err := lbV.Validate(data.(*jsonutils.JSONDict))
+	if err == nil {
+		return lbV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerId)
 	err := lbV.Validate(data)
@@ -925,12 +934,29 @@ func (man *SLoadbalancerBackendGroupManager) initBackendGroupType() error {
 }
 
 func (man *SLoadbalancerBackendGroupManager) InitializeData() error {
-	if err := man.initBackendGroupType(); err != nil {
-		return err
+	q := man.Query().IsNullOrEmpty("loadbalancer_id")
+	lbbgs := make([]SLoadbalancerBackendGroup, 0)
+	err := db.FetchModelObjects(man, q, &lbbgs)
+	if err != nil {
+		return errors.Wrap(err, "SLoadbalancerBackendGroupManager.InitializeData")
 	}
-	return man.initBackendGroupRegion()
+
+	for i := range lbbgs {
+		lbbg := lbbgs[i]
+		_, err = db.UpdateWithLock(context.Background(), &lbbg, func() error {
+			lbbg.MarkDelete()
+			return nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "SLoadbalancerBackendGroupManager.InitializeData.MarkDelete")
+		}
+	}
+
+	log.Debugf("SLoadbalancerBackendGroupManager.InitializeData removed %d invalid loadbalancer backendgroup.", len(lbbgs))
+	return nil
 }
 
+/*
 func (manager *SLoadbalancerBackendGroupManager) initBackendGroupRegion() error {
 	groups := []SLoadbalancerBackendGroup{}
 	q := manager.Query()
@@ -952,7 +978,7 @@ func (manager *SLoadbalancerBackendGroupManager) initBackendGroupRegion() error
 		}
 	}
 	return nil
-}
+}*/
 
 func (manager *SLoadbalancerBackendGroupManager) GetResourceCount() ([]db.SProjectResourceCount, error) {
 	virts := manager.Query().IsFalse("pending_deleted")

pkg/compute/models/loadbalancerbackends.go

@@ -146,6 +146,14 @@ func (man *SLoadbalancerBackendManager) ValidateBackendVpc(lb *SLoadbalancer, gu
 	return nil
 }
 
+func (man *SLoadbalancerBackendManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", nil)
+	if err := backendGroupV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return backendGroupV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerId)
 	if err := backendGroupV.Validate(data); err != nil {

pkg/compute/models/loadbalancerlistenerrules.go

@@ -397,6 +397,14 @@ func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context,
 	return q, nil
 }
 
+func (man *SLoadbalancerListenerRuleManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", nil)
+	if err := listenerV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return listenerV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	input := apis.VirtualResourceCreateInput{}
 	err := data.Unmarshal(&input)

pkg/compute/models/loadbalancerlisteners.go

@@ -234,6 +234,14 @@ func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q *
 	return q, nil
 }
 
+func (man *SLoadbalancerListenerManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", nil)
+	if err := lbV.Validate(data.(*jsonutils.JSONDict)); err == nil {
+		return lbV.Model.GetOwnerId(), nil
+	}
+	return man.SVirtualResourceBaseManager.FetchOwnerId(ctx, data)
+}
+
 func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerId)
 	if err := lbV.Validate(data); err != nil {
@@ -355,7 +363,7 @@ func (lblis *SLoadbalancerListener) StartLoadBalancerListenerSyncstatusTask(ctx
 func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	ownerId := lblis.GetOwnerId()
 	backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerId)
-	backendGroupV.AllowEmpty(true).Optional(true)
+	backendGroupV.AllowEmpty(true).Default(lblis.BackendGroupId)
 	if err := backendGroupV.Validate(data); err != nil {
 		return nil, err
 	}

pkg/compute/models/loadbalancers.go

@@ -798,11 +798,18 @@ func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred
 		lb.LoadbalancerSpec = extLb.GetLoadbalancerSpec()
 		lb.EgressMbps = extLb.GetEgressMbps()
 		lb.ChargeType = extLb.GetChargeType()
-
+		lbNetworkIds := getExtLbNetworkIds(extLb)
+		lb.NetworkId = strings.Join(lbNetworkIds, ",")
 		if extLb.GetMetadata() != nil {
 			lb.LBInfo = extLb.GetMetadata()
 		}
 
+		if vpcId := extLb.GetVpcId(); len(vpcId) > 0 {
+			if vpc, err := db.FetchByExternalId(VpcManager, vpcId); err == nil && vpc != nil {
+				lb.VpcId = vpc.GetId()
+			}
+		}
+
 		return nil
 	})
 

pkg/compute/models/networks.go

@@ -50,10 +50,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
-var (
-	ALL_NETWORK_TYPES = api.ALL_NETWORK_TYPES
-)
-
 type SNetworkManager struct {
 	db.SSharableVirtualResourceBaseManager
 }
@@ -752,10 +748,11 @@ func (manager *SNetworkManager) GetOnPremiseNetworkOfIP(ipAddr string, serverTyp
 	}
 	q := manager.Query()
 	wires := WireManager.Query().SubQuery()
-	vpcs := VpcManager.Query().SubQuery()
+	// vpcs := VpcManager.Query().SubQuery()
 	q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id")))
-	q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
-	q = q.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
+	// q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
+	// q = q.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
+	q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), api.DEFAULT_VPC_ID))
 	if len(serverType) > 0 {
 		q = q.Filter(sqlchemy.Equals(q.Field("server_type"), serverType))
 	}
@@ -1221,6 +1218,11 @@ func (manager *SNetworkManager) validateEnsureWire(ctx context.Context, userCred
 }
 
 func (manager *SNetworkManager) validateEnsureZoneVpc(ctx context.Context, userCred mcclient.TokenCredential, input api.NetworkCreateInput) (w *SWire, v *SVpc, cr *SCloudregion, err error) {
+	defer func() {
+		if cause := errors.Cause(err); cause == sql.ErrNoRows {
+			err = httperrors.NewResourceNotFoundError("%s", err)
+		}
+	}()
 	zObj, err := ZoneManager.FetchByIdOrName(userCred, input.Zone)
 	if err != nil {
 		err = errors.Wrapf(err, "zone %s", input.Zone)
@@ -1269,7 +1271,7 @@ func (manager *SNetworkManager) validateEnsureZoneVpc(ctx context.Context, userC
 func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.NetworkCreateInput) (api.NetworkCreateInput, error) {
 	if input.ServerType == "" {
 		input.ServerType = api.NETWORK_TYPE_GUEST
-	} else if !utils.IsInStringArray(input.ServerType, ALL_NETWORK_TYPES) {
+	} else if !utils.IsInStringArray(input.ServerType, api.ALL_NETWORK_TYPES) {
 		return input, httperrors.NewInputParameterError("Invalid server_type: %s", input.ServerType)
 	}
 
@@ -1810,8 +1812,17 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		}
 	}
 
+	ip := ""
+	exactIpMatch := false
 	if len(input.Ip) > 0 {
-		ipIa, err := parseIpToIntArray(input.Ip)
+		exactIpMatch = true
+		ip = input.Ip
+	} else if len(input.IpMatch) > 0 {
+		ip = input.IpMatch
+	}
+
+	if len(ip) > 0 {
+		ipIa, err := parseIpToIntArray(ip)
 		if err != nil {
 			return nil, err
 		}
@@ -1826,7 +1837,13 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		ipStart := sqlchemy.INET_ATON(q.Field("guest_ip_start"))
 		ipEnd := sqlchemy.INET_ATON(q.Field("guest_ip_end"))
 
-		ipCondtion := sqlchemy.OR(sqlchemy.Between(ipField, ipStart, ipEnd), sqlchemy.Contains(q.Field("guest_ip_start"), input.Ip), sqlchemy.Contains(q.Field("guest_ip_end"), input.Ip))
+		var ipCondtion sqlchemy.ICondition
+		if exactIpMatch {
+			ipCondtion = sqlchemy.Between(ipField, ipStart, ipEnd)
+		} else {
+			ipCondtion = sqlchemy.OR(sqlchemy.Between(ipField, ipStart, ipEnd), sqlchemy.Contains(q.Field("guest_ip_start"), ip), sqlchemy.Contains(q.Field("guest_ip_end"), ip))
+		}
+
 		q = q.Filter(ipCondtion)
 	}
 

pkg/compute/models/secgroupcache.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SSecurityGroupCacheManager struct {
@@ -107,6 +108,25 @@ func (self *SSecurityGroupCache) GetIRegion() (cloudprovider.ICloudRegion, error
 	return nil, fmt.Errorf("failed to find iregion for secgroupcache %s vpc: %s externalId: %s", self.Id, self.VpcId, self.ExternalId)
 }
 
+func (manager *SSecurityGroupCacheManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
+	if userCred != nil {
+		sq := SecurityGroupManager.Query("id")
+		switch scope {
+		case rbacutils.ScopeProject:
+			if len(userCred.GetProjectId()) > 0 {
+				sq = sq.Equals("tenant_id", userCred.GetProjectId())
+				return q.In("secgroup_id", sq)
+			}
+		case rbacutils.ScopeDomain:
+			if len(userCred.GetProjectDomainId()) > 0 {
+				sq = sq.Equals("domain_id", userCred.GetProjectDomainId())
+				return q.In("secgroup_id", sq)
+			}
+		}
+	}
+	return q
+}
+
 func (self *SSecurityGroupCache) GetVpc() (*SVpc, error) {
 	vpc, err := VpcManager.FetchById(self.VpcId)
 	if err != nil {

pkg/compute/models/secgrouprules.go

@@ -113,7 +113,7 @@ func (manager *SSecurityGroupRuleManager) FetchOwnerId(ctx context.Context, data
 		}
 		return secgroup.(*SSecurityGroup).GetOwnerId(), nil
 	}
-	return nil, nil
+	return db.FetchProjectInfo(ctx, data)
 }
 
 func (manager *SSecurityGroupRuleManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {

pkg/compute/models/secgroups.go

@@ -711,18 +711,21 @@ func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mc
 	} else {
 		needSync := false
 
-		lockman.LockObject(ctx, secgrp)
-		defer lockman.ReleaseObject(ctx, secgrp)
+		func() {
+			lockman.LockObject(ctx, secgrp)
+			defer lockman.ReleaseObject(ctx, secgrp)
 
-		if secgrp.IsDirty {
-			if _, err := db.Update(secgrp, func() error {
-				secgrp.IsDirty = false
-				return nil
-			}); err != nil {
-				log.Errorf("Update Security Group error: %s", err.Error())
+			if secgrp.IsDirty {
+				if _, err := db.Update(secgrp, func() error {
+					secgrp.IsDirty = false
+					return nil
+				}); err != nil {
+					log.Errorf("Update Security Group error: %s", err.Error())
+				}
+				needSync = true
 			}
-			needSync = true
-		}
+		}()
+
 		if needSync {
 			for _, guest := range secgrp.GetGuests() {
 				guest.StartSyncTask(ctx, userCred, true, "")
@@ -739,7 +742,7 @@ func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenC
 		log.Errorf("Update Security Group error: %s", err.Error())
 	}
 	time.AfterFunc(10*time.Second, func() {
-		SecurityGroupManager.DelaySync(ctx, userCred, self.Id)
+		SecurityGroupManager.DelaySync(context.Background(), userCred, self.Id)
 	})
 }
 

pkg/compute/models/snapshots.go

@@ -363,9 +363,8 @@ func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string)
 
 func (self *SSnapshotManager) GetDiskSnapshots(diskId string) []SSnapshot {
 	dest := make([]SSnapshot, 0)
-	q := self.Query().SubQuery()
-	sq := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId)))
-	err := db.FetchModelObjects(self, sq, &dest)
+	q := self.Query().Equals("disk_id", diskId).Asc("created_at")
+	err := db.FetchModelObjects(self, q, &dest)
 	if err != nil {
 		log.Errorf("GetDiskSnapshots error: %s", err)
 		return nil

pkg/compute/models/wires.go

@@ -116,8 +116,8 @@ func (manager *SWireManager) ValidateCreateData(ctx context.Context, userCred mc
 			break
 		}
 	}
-	if len(input.Vpc) == 0 {
-		return nil, httperrors.NewMissingParameterError("vpc")
+	if input.Vpc == "" {
+		input.Vpc = api.DEFAULT_VPC_ID
 	}
 
 	_vpc, err := VpcManager.FetchByIdOrName(userCred, input.Vpc)
@@ -546,6 +546,7 @@ func (manager *SWireManager) totalCountQ(
 	wires := WireManager.Query().SubQuery()
 	q := wires.Query(
 		sqlchemy.COUNT("id").Label("wires_count"),
+		sqlchemy.SUM("emulated_wires_count", wires.Field("is_emulated")),
 		sqlchemy.SUM("net_count", netSQ.Field("net_count")),
 		sqlchemy.SUM("guest_nic_count", netSQ.Field("gnic_count")),
 		sqlchemy.SUM("host_nic_count", netSQ.Field("hnic_count")),
@@ -576,13 +577,14 @@ func (manager *SWireManager) totalCountQ(
 }
 
 type WiresCountStat struct {
-	WiresCount    int
-	NetCount      int
-	GuestNicCount int
-	HostNicCount  int
-	ReservedCount int
-	GroupNicCount int
-	LbNicCount    int
+	WiresCount         int
+	EmulatedWiresCount int
+	NetCount           int
+	GuestNicCount      int
+	HostNicCount       int
+	ReservedCount      int
+	GroupNicCount      int
+	LbNicCount         int
 }
 
 func (wstat WiresCountStat) NicCount() int {

pkg/compute/regiondrivers/aws.go

@@ -103,7 +103,7 @@ func validateAwsLbNetwork(ownerId mcclient.IIdentityProvider, data *jsonutils.JS
 				var nets []models.SNetwork
 				wires := models.WireManager.Query().SubQuery()
 				q := models.NetworkManager.Query().IsFalse("pending_deleted")
-				q = models.NetworkManager.FilterByOwner(q, network.GetOwnerId(), rbacutils.ScopeProject)
+				q = models.NetworkManager.FilterByOwner(q, ownerId, rbacutils.ScopeProject)
 				q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id")))
 				q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), vpc.GetId()))
 				q = q.Filter(sqlchemy.NotEquals(wires.Field("zone_id"), zone.GetId()))

pkg/compute/regiondrivers/managedvirtual.go

@@ -1490,8 +1490,11 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 			secgroup.Name = "DefaultGroup"
 		}
 		// 避免有的云不支持重名安全组
-		groupName := secgroup.Name
-		for i := 0; i < 30; i++ {
+		randomString := func(prefix string, length int) string {
+			return fmt.Sprintf("%s-%s", prefix, rand.String(length))
+		}
+		groupName := randomString(secgroup.Name, 1)
+		for i := 2; i < 30; i++ {
 			_, err := iRegion.GetISecurityGroupByName(vpc.ExternalId, groupName)
 			if err != nil {
 				if errors.Cause(err) == cloudprovider.ErrNotFound {
@@ -1501,7 +1504,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 					return "", err
 				}
 			}
-			groupName = fmt.Sprintf("%s-%d", secgroup.Name, i)
+			groupName = randomString(secgroup.Name, i)
 		}
 		conf := &cloudprovider.SecurityGroupCreateInput{
 			Name:  groupName,
@@ -1547,7 +1550,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 	sort.Sort(outRules)
 	_inAllowList := inRules.AllowList()
 	_outAllowList := outRules.AllowList()
-	if inAllowList.Equals(_inAllowList) && outAllowList.Equals(_outAllowList) {
+	if inAllowList.Equals(_inAllowList) && outAllowList.Equals(_outAllowList) && (len(_inAllowList) > 0 && len(_outAllowList) > 0) { // 避免单个deny any的allowList为空,导致安全组规则未同步
 		return cache.ExternalId, nil
 	}
 
@@ -1665,37 +1668,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstance(ctx cont
 			log.Errorf("timeout for waiting dbinstance running error: %v", err)
 		}
 
-		dbinstance.ZoneId = idbinstance.GetIZoneId()
-		err = dbinstance.SetZoneInfo(ctx, userCred)
-		if err != nil {
-			log.Errorf("failed to set dbinstance %s(%s) zoneInfo from cloud dbinstance: %v", dbinstance.Name, dbinstance.Id, err)
-		}
-
-		_, err = db.Update(dbinstance, func() error {
-			dbinstance.Engine = idbinstance.GetEngine()
-			dbinstance.EngineVersion = idbinstance.GetEngineVersion()
-			dbinstance.StorageType = idbinstance.GetStorageType()
-			dbinstance.DiskSizeGB = idbinstance.GetDiskSizeGB()
-			dbinstance.Category = idbinstance.GetCategory()
-			dbinstance.VcpuCount = idbinstance.GetVcpuCount()
-			dbinstance.VmemSizeMb = idbinstance.GetVmemSizeMB()
-			dbinstance.InstanceType = idbinstance.GetInstanceType()
-			dbinstance.ConnectionStr = idbinstance.GetConnectionStr()
-			dbinstance.InternalConnectionStr = idbinstance.GetInternalConnectionStr()
-			dbinstance.MaintainTime = idbinstance.GetMaintainTime()
-			dbinstance.Port = idbinstance.GetPort()
-
-			if createdAt := idbinstance.GetCreatedAt(); !createdAt.IsZero() {
-				dbinstance.CreatedAt = idbinstance.GetCreatedAt()
-			}
-			if expiredAt := idbinstance.GetExpiredAt(); !expiredAt.IsZero() {
-				dbinstance.ExpiredAt = expiredAt
-			}
-			return nil
-		})
-		if err != nil {
-			log.Errorf("failed to update dbinstance conf: %v", err)
-		}
+		dbinstance.SyncWithCloudDBInstance(ctx, userCred, dbinstance.GetCloudprovider(), idbinstance)
 
 		network, err := idbinstance.GetDBNetwork()
 		if err != nil {

pkg/compute/service/service.go

@@ -54,15 +54,14 @@ func StartService() {
 	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
+	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
 	app := app_common.InitApp(baseOpts, true)
-	InitHandlers(app)
 
+	InitHandlers(app)
 	db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
 	defer cloudcommon.CloseDB()
 
-	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
-
 	options.InitNameSyncResources()
 
 	err := setInfluxdbRetentionPolicy()

pkg/compute/tasks/guest_live_migrate_task.go

@@ -105,6 +105,17 @@ func (self *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj ISched
 	isLocalStorage := utils.IsInStringArray(disk.GetStorage().StorageType,
 		api.STORAGE_LOCAL_TYPES)
 	if isLocalStorage {
+		targetStorages := jsonutils.NewArray()
+		for i := 0; i < len(disks); i++ {
+			var targetStroage string
+			if len(target.Disks[i].StorageIds) == 0 {
+				targetStroage = targetHost.GetLeastUsedStorage(disk.GetStorage().StorageType).Id
+			} else {
+				targetStroage = target.Disks[i].StorageIds[0]
+			}
+			targetStorages.Add(jsonutils.NewString(targetStroage))
+		}
+		body.Set("target_storages", targetStorages)
 		body.Set("is_local_storage", jsonutils.JSONTrue)
 	} else {
 		body.Set("is_local_storage", jsonutils.JSONFalse)
@@ -309,19 +320,14 @@ func (self *GuestMigrateTask) localStorageMigrateConf(ctx context.Context,
 		self.TaskFailed(ctx, guest, "Get disksDesc error")
 		return nil, true
 	}
-	targetStorageId, _ := disksDesc[0].GetString("target_storage_id")
-	if len(targetStorageId) == 0 {
-		self.TaskFailed(ctx, guest, "Get targetStorageId error")
-		return nil, true
+	targetStorages, _ := self.Params.GetArray("target_storages")
+	for i := 0; i < len(disks); i++ {
+		diskDesc := disksDesc[i].(*jsonutils.JSONDict)
+		diskDesc.Set("target_storage_id", targetStorages[i])
 	}
 
-	targetStorage := targetHost.GetHoststorageOfId(targetStorageId)
-	sourceStorage := sourceHost.GetHoststorageOfId(disks[0].GetDisk().StorageId)
-	if sourceStorage.MountPoint != targetStorage.MountPoint {
-		// rebase disks backing file
-		body.Set("rebase_disks", jsonutils.JSONTrue)
-	}
 	body.Set("desc", targetDesc)
+	body.Set("rebase_disks", jsonutils.JSONTrue)
 	body.Set("is_local_storage", jsonutils.JSONTrue)
 	return body, false
 }
@@ -363,20 +369,19 @@ func (self *GuestLiveMigrateTask) OnStartDestCompleteFailed(ctx context.Context,
 func (self *GuestMigrateTask) setGuest(ctx context.Context, guest *models.SGuest) error {
 	targetHostId, _ := self.Params.GetString("target_host_id")
 	if jsonutils.QueryBoolean(self.Params, "is_local_storage", false) {
-		targetHost := models.HostManager.FetchHostById(targetHostId)
-		targetStorage := targetHost.GetLeastUsedStorage(api.STORAGE_LOCAL)
+		targetStorages, _ := self.Params.GetArray("target_storages")
 		guestDisks := guest.GetDisks()
 		for i := 0; i < len(guestDisks); i++ {
 			disk := guestDisks[i].GetDisk()
 			db.Update(disk, func() error {
 				disk.Status = api.DISK_READY
-				disk.StorageId = targetStorage.Id
+				disk.StorageId, _ = targetStorages[i].GetString()
 				return nil
 			})
 			snapshots := models.SnapshotManager.GetDiskSnapshots(disk.Id)
 			for _, snapshot := range snapshots {
 				db.Update(&snapshot, func() error {
-					snapshot.StorageId = targetStorage.Id
+					snapshot.StorageId, _ = targetStorages[i].GetString()
 					return nil
 				})
 			}

pkg/compute/usages/handler.go

@@ -419,7 +419,7 @@ func ZoneUsage(providers []string, brands []string, cloudEnv string) Usage {
 }
 
 func VpcUsage(providers []string, brands []string, cloudEnv string) Usage {
-	q := models.VpcManager.Query()
+	q := models.VpcManager.Query().IsFalse("is_emulated")
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		q = models.CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv)
 	}
@@ -514,7 +514,7 @@ func DisksUsage(
 func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage {
 	count := make(map[string]interface{})
 	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil, false)
-	count["wires"] = result.WiresCount
+	count["wires"] = result.WiresCount - result.EmulatedWiresCount
 	count["networks"] = result.NetCount
 	count["all.nics.guest"] = result.GuestNicCount
 	count["all.nics.host"] = result.HostNicCount

pkg/hostman/diskutils/vddk.go

@@ -115,8 +115,8 @@ func (c *Command) Wait() error {
 	return <-c.done
 }
 
-func (c *Command) Kill() {
-	c.Process.Kill()
+func (c *Command) Kill() error {
+	return c.Process.Kill()
 }
 
 func execpath() string {
@@ -154,7 +154,12 @@ func (vd *VDDKDisk) MountRootfs() fsdriver.IRootFsDriver {
 		log.Errorf("VDDKDisk Mount failed: %s", err)
 	}
 	// something is wrong
-	vd.UmountRootfs(nil)
+	if vd.Proc != nil {
+		err := vd.Proc.Kill()
+		if err != nil {
+			log.Errorf("unable to kill proc: %s", err.Error())
+		}
+	}
 	return nil
 }
 
@@ -336,6 +341,7 @@ Loop:
 	}
 
 	backup := vd.Proc.stdouterr.String()
+	log.Debugf(backup)
 	err := vd.ParsePartitions(backup)
 	if err != nil {
 		return errors.Wrap(err, "VDDKDisk.ParsePartitions")

pkg/hostman/guestfs/sshpart/sshpart.go

@@ -309,20 +309,25 @@ func (p *SSHPartition) sshFilePutContents(sPath, content string, modAppend bool)
 	}
 
 	cmds := []string{}
-	var chunkSize int = 8192
-	for offset := 0; offset < len(content); offset += chunkSize {
-		end := offset + chunkSize
-		if end > len(content) {
-			end = len(content)
-		}
-		ll, err := stringutils.EscapeEchoString(content[offset:end])
-		if err != nil {
-			return fmt.Errorf("EscapeEchoString %q error: %v", content[offset:end], err)
-		}
-		cmd := fmt.Sprintf(`echo -n -e "%s" %s %s`, ll, op, sPath)
+	if len(content) == 0 {
+		cmd := fmt.Sprintf(`echo -n -e "" %s %s`, op, sPath)
 		cmds = append(cmds, cmd)
-		if op == ">" {
-			op = ">>"
+	} else {
+		var chunkSize int = 8192
+		for offset := 0; offset < len(content); offset += chunkSize {
+			end := offset + chunkSize
+			if end > len(content) {
+				end = len(content)
+			}
+			ll, err := stringutils.EscapeEchoString(content[offset:end])
+			if err != nil {
+				return fmt.Errorf("EscapeEchoString %q error: %v", content[offset:end], err)
+			}
+			cmd := fmt.Sprintf(`echo -n -e "%s" %s %s`, ll, op, sPath)
+			cmds = append(cmds, cmd)
+			if op == ">" {
+				op = ">>"
+			}
 		}
 	}
 	_, err := p.term.Run(cmds...)

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -332,11 +332,17 @@ func guestDestPrepareMigrate(ctx context.Context, sid string, body jsonutils.JSO
 		if err != nil {
 			return nil, httperrors.NewInputParameterError("Get desc disks error")
 		} else {
-			targetStorageId, _ := disks[0].GetString("target_storage_id")
-			if len(targetStorageId) == 0 {
-				return nil, httperrors.NewMissingParameterError("target_storage_id")
+			targetStorageIds := []string{}
+			for i := 0; i < len(disks); i++ {
+				targetStorageId, _ := disks[i].GetString("target_storage_id")
+				if len(targetStorageId) == 0 {
+					return nil, httperrors.NewMissingParameterError("target_storage_id")
+				}
+				targetStorageIds = append(targetStorageIds, targetStorageId)
+				// params.TargetStorageId = targetStorageId
+				params.TargetStorageIds = targetStorageIds
 			}
-			params.TargetStorageId = targetStorageId
+
 		}
 		params.RebaseDisks = jsonutils.QueryBoolean(body, "rebase_disks", false)
 	}

pkg/hostman/guestman/guesthelper.go

@@ -37,14 +37,15 @@ type SSrcPrepareMigrate struct {
 }
 
 type SDestPrepareMigrate struct {
-	Sid             string
-	ServerUrl       string
-	QemuVersion     string
-	SnapshotsUri    string
-	DisksUri        string
-	TargetStorageId string
-	LiveMigrate     bool
-	RebaseDisks     bool
+	Sid          string
+	ServerUrl    string
+	QemuVersion  string
+	SnapshotsUri string
+	DisksUri     string
+	// TargetStorageId string
+	TargetStorageIds []string
+	LiveMigrate      bool
+	RebaseDisks      bool
 
 	Desc             jsonutils.JSONObject
 	DisksBackingFile jsonutils.JSONObject