Merge pull request #9114 from yousong/automated-cherry-pick-of-#9109-upstream-release-3.2

Automated cherry pick of #9109: fix(host): make sure it won't match vpc guests by accident

build/docker/Dockerfile.ansibleserver

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:1.0.0
 
-ADD ./_output/bin/ansibleserver /opt/yunion/bin/ansibleserver
+ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -1,13 +1,11 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
+FROM alpine:3.11.6
 
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk update && \
-    apk add --no-cache tzdata ansible ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+# openssh-client, for ansible ssh connection
+RUN set -x \
+	&& apk update \
+	&& apk add openssh-client \
+	&& apk add ansible \
+	&& apk add tzdata ca-certificates \
+	&& rm -rf /var/cache/apk/*

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/logger /opt/yunion/bin/logger
 

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/monitor /opt/yunion/bin/monitor
 

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -7,7 +7,7 @@ ENV TZ Asia/Shanghai
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates && \
+    apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.yunionconf

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/yunionconf /opt/yunion/bin/yunionconf
 

build/notify/root/opt/yunion/share/template/content/VERIFY.email

@@ -11,7 +11,7 @@
   			<table style="margin-left: 20px;">
   				<tr>
   					<td>
-  						<img src="data:{{.logo_format}};base64,{{.logo}}" alt="" style="color: #fff; width: 80px; vertical-align: middle;">
+  						<img src="data:{{.login_logo_format}};base64,{{.login_logo}}" alt="" style="color: #fff; height: 32px; vertical-align: middle;">
   					</td>
   				</tr>
   			</table>
@@ -56,10 +56,10 @@
   		</td>
   	</tr>
   	<tr style="width: 96%;">
-		<td colspan="2" style="border-top: 1px dashed #ccc; color: #ccc; font-size: 12px; padding-bottom: 10px; font-weight: 100;">*为了确保您的帐号安全，该链接仅48小时内访问有效，请勿直接回复此邮件。</td>
+		<td colspan="2" style="border-top: 1px dashed #ccc; color: #333; font-size: 12px; padding-bottom: 10px; font-weight: 100;">*为了确保您的帐号安全，该链接仅48小时内访问有效，请勿直接回复此邮件。</td>
 	</tr>
 	<tr>
-		<td colspan="2" style="background: #333; text-align: right; padding-right: 20px; font-size: 12px; color: #949494; height: 50px;">版权所有 © {{.copyright}} 保留一切权利</td>
+		<td colspan="2" style="background: #333; text-align: right; padding-right: 20px; font-size: 12px; color: #fff; height: 50px;">版权所有 © {{.copyright}} 保留一切权利</td>
 	</tr>
   </table>
 </body>

cmd/climc/shell/events.go

@@ -35,7 +35,10 @@ type BaseEventListOptions struct {
 	Action     []string `help:"Log action"`
 
 	User    string `help:"filter by operator user"`
-	Project string `help:"filter by owner project"`
+	Project string `help:"filter by operator user's project"`
+
+	OwnerProjectIds []string `help:"filter by owner project ids"`
+	OwnerDomainIds  []string `help:"filter by owner domain ids"`
 
 	PagingMarker string `help:"marker for pagination"`
 }
@@ -107,6 +110,12 @@ func doEventList(man modulebase.ResourceManager, s *mcclient.ClientSession, args
 	if len(args.Scope) > 0 {
 		params.Add(jsonutils.NewString(args.Scope), "scope")
 	}
+	if len(args.OwnerProjectIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerProjectIds), "owner_project_ids")
+	}
+	if len(args.OwnerDomainIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerDomainIds), "owner_domain_ids")
+	}
 	if len(args.PagingMarker) > 0 {
 		params.Add(jsonutils.NewString(args.PagingMarker), "paging_marker")
 	}

cmd/climc/shell/globalvpcs.go

@@ -15,6 +15,8 @@
 package shell
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -78,4 +80,30 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	R(&GlobalVpcShowOptions{}, "global-vpc-change-owner-candidate-domains", "Show candiate domains of a global vpc for changing owner", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
+		result, err := modules.GlobalVpcs.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type GlobalVpcChangeOwnerOptions struct {
+		ID            string `help:"ID or name of vpc" json:"-"`
+		ProjectDomain string `json:"project_domain" help:"target domain"`
+	}
+	R(&GlobalVpcChangeOwnerOptions{}, "global-vpc-change-owner", "Change owner domain of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcChangeOwnerOptions) error {
+		if len(args.ProjectDomain) == 0 {
+			return fmt.Errorf("empty project_domain")
+		}
+		params := jsonutils.Marshal(args)
+		ret, err := modules.GlobalVpcs.PerformAction(s, args.ID, "change-owner", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/identityproviders.go

@@ -299,7 +299,9 @@ func init() {
 		IDP string `help:"identity provider name or ID"`
 	}
 	R(&IdentityProviderConfigEditOptions{}, "idp-config-edit", "Edit config yaml of an identity provider", func(s *mcclient.ClientSession, args *IdentityProviderConfigEditOptions) error {
-		conf, err := modules.IdentityProviders.GetSpecific(s, args.IDP, "config", nil)
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.JSONTrue, "sensitive")
+		conf, err := modules.IdentityProviders.GetSpecific(s, args.IDP, "config", params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/images.go

@@ -54,6 +54,7 @@ type ImageOptionalOptions struct {
 	Hypervisor         []string `help:"Prefer hypervisor type" choices:"kvm|esxi|baremetal|container|openstack|ctyun"`
 	DiskDriver         string   `help:"Perfer disk driver" choices:"virtio|scsi|pvscsi|ide|sata"`
 	NetDriver          string   `help:"Preferred network driver" choices:"virtio|e1000|vmxnet3"`
+	DisableUsbKbd      bool     `help:"Disable usb keyboard on this image(for hypervisor kvm)"`
 }
 
 func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDict, args ImageOptionalOptions) error {
@@ -131,6 +132,9 @@ func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDi
 	if len(args.Hypervisor) > 0 {
 		params.Add(jsonutils.NewString(strings.Join(args.Hypervisor, ",")), "properties", "hypervisor")
 	}
+	if args.DisableUsbKbd {
+		params.Add(jsonutils.NewString("true"), "properties", "disable_usb_kbd")
+	}
 	return nil
 }
 
@@ -252,7 +256,7 @@ func init() {
 	}
 
 	type ImageDeleteOptions struct {
-		ID                    []string `help:"Image ID or name"`
+		ID                    []string `help:"Image ID or name" json:"-"`
 		OverridePendingDelete *bool    `help:"Delete image directly instead of pending delete" short-token:"f"`
 	}
 	R(&ImageDeleteOptions{}, "image-delete", "Delete a image", func(s *mcclient.ClientSession, args *ImageDeleteOptions) error {

cmd/climc/shell/policies.go

@@ -288,12 +288,30 @@ func init() {
 		if err != nil {
 			log.Fatalf("Set log level %q: %v", "debug", err)
 		}
+		if args.Debug {
+			rbacutils.ShowMatchRuleDebug = true
+		}
 		auth.InitFromClientSession(s)
 		policy.EnableGlobalRbac(15*time.Second, 15*time.Second, false)
 		if args.Debug {
 			consts.EnableRbacDebug()
 		}
 
+		findPolicy := false
+		for !findPolicy {
+			all := policy.PolicyManager.AllPolicies()
+			for _, allP := range all {
+				if len(allP) > 0 {
+					findPolicy = true
+					break
+				}
+			}
+			if findPolicy {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
 		req := jsonutils.NewDict()
 		for i := 0; i < len(args.Request); i += 1 {
 			parts := strings.Split(args.Request[i], ":")
@@ -359,6 +377,7 @@ func init() {
 				Context: mcclient.SAuthContext{
 					Ip: args.Ip,
 				},
+				Token: "faketoken",
 			}
 		} else {
 			token = s.GetToken()
@@ -370,6 +389,10 @@ func init() {
 		}
 		printObject(result)
 
+		for _, r := range args.Role {
+			fmt.Println("role", r, "matched policies:", policy.PolicyManager.RoleMatchPolicies(r))
+		}
+
 		fmt.Println("userCred:", token)
 		for _, scope := range []rbacutils.TRbacScope{
 			rbacutils.ScopeSystem,

cmd/climc/shell/secgrouprules.go

@@ -25,33 +25,20 @@ import (
 func init() {
 	type SecGroupRulesListOptions struct {
 		options.BaseListOptions
-		Secgroup  string `help:"Secgroup ID or Name"`
-		Direction string `help:"filter Direction of rule" choices:"in|out"`
-		Protocol  string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Action    string `help:"filter Actin of rule" choices:"allow|deny"`
+		Secgroup     string   `help:"Secgroup ID or Name"`
+		SecgroupName string   `help:"Search rules by fuzzy secgroup name"`
+		Projects     []string `help:"Filter rules by project"`
+		Direction    string   `help:"filter Direction of rule" choices:"in|out"`
+		Protocol     string   `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
+		Action       string   `help:"filter Actin of rule" choices:"allow|deny"`
+		Ports        string   `help:"filter Ports of rule"`
+		Ip           string   `help:"filter cidr of rule"`
 	}
 
 	R(&SecGroupRulesListOptions{}, "secgroup-rule-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupRulesListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Secgroup) > 0 {
-			params.Add(jsonutils.NewString(args.Secgroup), "secgroup")
-		}
-		if len(args.Direction) > 0 {
-			params.Add(jsonutils.NewString(args.Direction), "direction")
-		}
-		if len(args.Protocol) > 0 {
-			params.Add(jsonutils.NewString(args.Protocol), "protocol")
-		}
-		if len(args.Action) > 0 {
-			params.Add(jsonutils.NewString(args.Action), "action")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroupRules.List(s, params)
 		if err != nil {

cmd/climc/shell/secgroups.go

@@ -29,23 +29,15 @@ func init() {
 		Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"`
 		Server string `help:"Filter secgroups bound to specified server"`
 		options.BaseListOptions
+		Ip        string `help:"Filter secgroup by ip"`
+		Ports     string `help:"Filter secgroup by ports"`
+		Direction string `help:"Filter secgroup by ports" choices:"all|in|out"`
 	}
 
 	R(&SecGroupsListOptions{}, "secgroup-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupsListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Equals) > 0 {
-			params.Add(jsonutils.NewString(args.Equals), "equals")
-		}
-		if len(args.Server) > 0 {
-			params.Add(jsonutils.NewString(args.Server), "server")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroups.List(s, params)
 		if err != nil {

cmd/climc/shell/servers.go

@@ -1146,6 +1146,7 @@ func init() {
 		ID               string `help:"ID or name of VM"`
 		DisableIsaSerial string `help:"disable isa serial device" choices:"true|false"`
 		DisablePvpanic   string `help:"disable pvpanic device" choices:"true|false"`
+		DisableUsbKbd    string `help:"disable usb kbd" choices:"true|false"`
 	}
 
 	R(&ServerQemuParams{}, "server-set-qemu-params", "config qemu params", func(s *mcclient.ClientSession,
@@ -1157,6 +1158,9 @@ func init() {
 		if len(opts.DisablePvpanic) > 0 {
 			params.Set("disable_pvpanic", jsonutils.NewString(opts.DisablePvpanic))
 		}
+		if len(opts.DisableUsbKbd) > 0 {
+			params.Set("disable_usb_kbd", jsonutils.NewString(opts.DisableUsbKbd))
+		}
 		result, err := modules.Servers.PerformAction(s, opts.ID, "set-qemu-params", params)
 		if err != nil {
 			return err

cmd/googlecli/main.go

@@ -18,17 +18,21 @@ import (
 	"fmt"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
 	Debug        bool   `help:"debug mode"`
 	Help         bool   `help:"Show help"`
+	AuthFile     string `help:"google cloud auth json file path" default:"$GOOGLE_AUTH_FILE"`
 	ClientEmail  string `help:"Client email" default:"$GOOGLE_CLIENT_EMAIL"`
 	ProjectID    string `help:"Project ID" default:"$GOOGLE_PROJECT_ID"`
 	PrivateKeyID string `help:"Private Key ID" default:"$GOOGLE_PRIVATE_KEY_ID"`
@@ -78,6 +82,20 @@ func showErrorAndExit(e error) {
 }
 
 func newClient(options *BaseOptions) (*google.SRegion, error) {
+	if len(options.AuthFile) > 0 {
+		jsonStr, err := fileutils2.FileGetContents(options.AuthFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "FileGetContents")
+		}
+		jsonCfg, err := jsonutils.ParseString(jsonStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.ParseString")
+		}
+		options.ClientEmail, _ = jsonCfg.GetString("client_email")
+		options.PrivateKeyID, _ = jsonCfg.GetString("private_key_id")
+		options.PrivateKey, _ = jsonCfg.GetString("private_key")
+		options.ProjectID, _ = jsonCfg.GetString("project_id")
+	}
 	if len(options.ClientEmail) == 0 {
 		return nil, fmt.Errorf("Missing ClientEmail")
 	}

go.mod

@@ -117,6 +117,7 @@ require (
 	google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb
 	google.golang.org/grpc v1.26.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/fatih/set.v0 v0.2.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
@@ -127,12 +128,12 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501
+	yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
-	yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa
+	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
-	yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e
+	yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6
+	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 
 replace (

go.sum

@@ -1013,6 +1013,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fatih/set.v0 v0.2.1 h1:Xvyyp7LXu34P0ROhCyfXkmQCAoOUKb1E2JS9I7SE5CY=
+gopkg.in/fatih/set.v0 v0.2.1/go.mod h1:5eLWEndGL4zGGemXWrKuts+wTJR0y+w+auqUJZbmyBg=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -1106,8 +1108,8 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501 h1:i1r9XvbdxH3FgTCLmTaRi3MzQqhiQimXJRlUOPgrxnU=
-yunion.io/x/jsonutils v0.0.0-20200415132054-2bf8a5e94501/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8 h1:KQVw5EmohJCqBDMdZ3fAcOQNNMIYgVuNCNWNwg4EM3A=
+yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
@@ -1115,12 +1117,11 @@ yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960Ru
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa h1:VizPfW8+mLFEE7W/97zxQJbfdxchi2dn1M/Y7YBwTTc=
-yunion.io/x/pkg v0.0.0-20200516092703-0a53bc9270aa/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
+yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
-yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e h1:pctCe/EPel3F1B83pJ2q9b34Umd1NdbLW1Yd+Lzur2s=
-yunion.io/x/structarg v0.0.0-20200423163001-168d0687be7e/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6 h1:k7E8Zjwzyk/lo1dDO0jtLbODwYRNv+w/zsUJrKWCXvo=
+yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/service/handlers.go

@@ -39,6 +39,7 @@ func InitHandlers(app *appsrv.Application) {
 	db.InitAllManagers()
 
 	db.RegisterModelManager(db.OpsLog)
+	db.RegisterModelManager(db.Metadata)
 	db.RegisterModelManager(db.UserCacheManager)
 	db.RegisterModelManager(db.TenantCacheManager)
 	for _, manager := range []db.IModelManager{

pkg/apigateway/handler/auth.go

@@ -42,6 +42,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/netutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 func AppContextToken(ctx context.Context) mcclient.TokenCredential {
@@ -271,7 +272,7 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 			return nil, httperrors.NewInputParameterError("get password in body")
 		}
 		// try base64 decryption
-		if decPasswd, err := base64.StdEncoding.DecodeString(passwd); err == nil {
+		if decPasswd, err := base64.StdEncoding.DecodeString(passwd); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
 			passwd = string(decPasswd)
 		}
 		if len(uname) == 0 || len(passwd) == 0 {
@@ -395,7 +396,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
+		httperrors.ForbiddenError(w, "user forbidden login from web")
 		return false
 	}
 	return true

pkg/apigateway/handler/misc.go

@@ -28,6 +28,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -57,6 +59,12 @@ const (
 	BATCH_HOST_REGISTER_QUANTITY_LIMITATION = 1000
 )
 
+var (
+	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
+	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+)
+
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
 	token := AppContextToken(ctx)
 	session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion)
@@ -170,20 +178,43 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	// check header line
+	titlesOk := false
+	for _, t := range [][]string{BatchHostRegisterTemplate, BatchHostISORegisterTemplate, BatchHostPXERegisterTemplate} {
+		if len(t) == len(rows[0]) {
+			for _, title := range rows[0] {
+				if !utils.IsInStringArray(title, t) {
+					break
+				}
+			}
+
+			titlesOk = true
+		}
+	}
+
+	if !titlesOk {
+		httperrors.InputParameterError(w, "template file is invalid.please check.")
+		return
+	}
+
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +230,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 
@@ -297,18 +353,6 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			return
 		}
 
-		if _, ok := names[name]; ok {
-			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
-			httperrors.JsonClientError(w, e)
-			return
-		} else {
-			names[name] = true
-			_, err := modules.UsersV3.Get(s, name, nil)
-			if err == nil {
-				continue
-			}
-		}
-
 		domainId, ok := domains[domain]
 		if !ok {
 			if len(domain) == 0 {
@@ -327,6 +371,18 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			domains[domain] = id
 		}
 
+		if _, ok := names[name+"/"+domainId]; ok {
+			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
+			httperrors.JsonClientError(w, e)
+			return
+		} else {
+			names[name+"/"+domainId] = true
+			_, err := modules.UsersV3.Get(s, name, nil)
+			if err == nil {
+				continue
+			}
+		}
+
 		user := jsonutils.NewDict()
 		user.Add(jsonutils.NewString(name), "name")
 		user.Add(jsonutils.NewString(domainId), "domain_id")
@@ -375,21 +431,21 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 	var content bytes.Buffer
 	switch template {
 	case "BatchHostRegister":
-		records := [][]string{{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}}
+		records := [][]string{BatchHostRegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostISORegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}}
+		records := [][]string{BatchHostISORegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostPXERegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}}
+		records := [][]string{BatchHostPXERegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")

pkg/apigateway/options/options.go

@@ -26,7 +26,7 @@ type GatewayOptions struct {
 
 	DisableModuleApiVersion bool `help:"Disable each modules default api version" default:"false"`
 
-	EnableTotp bool `help:"Enable two-factor authentication"  default:"false"`
+	EnableTotp bool `help:"Enable two-factor authentication" default:"false"`
 
 	SqlitePath string `help:"sqlite db path" default:"/etc/yunion/data/yunionapi.db"`
 

pkg/apis/cloudcommon/proxy/proxy.go

@@ -57,7 +57,7 @@ type ProxySettingResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
+	ProxySettingId string `json:"proxy_setting_id" yunion-deprecated-by:"proxy_setting"`
 }
 
 type ProxySettingTestInput struct {

pkg/apis/compute/api.go

@@ -285,7 +285,7 @@ type ServerConfigs struct {
 	// swagger:ignore
 	// Deprecated
 	// alias for InstanceType
-	Sku string `json:"sku" "yunion:deprecated-by":"instance_type"`
+	Sku string `json:"sku" yunion-deprecated-by:"instance_type"`
 
 	// 虚拟机高可用(创建备机)
 	// default: false
@@ -359,7 +359,7 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	KeypairId string `json:"keypair_id" "yunion:deprecated-by":"keypair"`
+	KeypairId string `json:"keypair_id" yunion-deprecated-by:"keypair"`
 
 	// 秘钥对Id
 	// required: false
@@ -466,7 +466,8 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	OsType string `json:"os_type"`
-
+	// swagger:ignore
+	DisableUsbKbd bool `json:"disable_usb_kbd"`
 	// swagger:ignore
 	OsProfile jsonutils.JSONObject `json:"__os_profile__"`
 	// swagger:ignore

pkg/apis/compute/cachedloadbalanceracl.go

@@ -42,7 +42,7 @@ type LoadbalancerAclResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	AclId string `json:"acl_id" "yunion:deprecated-by":"acl"`
+	AclId string `json:"acl_id" yunion-deprecated-by:"acl"`
 }
 
 type LoadbalancerAclFilterListInput struct {

pkg/apis/compute/cachedloadbalancercertificate.go

@@ -43,7 +43,7 @@ type LoadbalancerCertificateResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	CertificateId string `json:"certificate_id" "yunion:deprecated-by":"certificate"`
+	CertificateId string `json:"certificate_id" yunion-deprecated-by:"certificate"`
 }
 
 type LoadbalancerCertificateFilterListInput struct {

pkg/apis/compute/cloudaccount.go

@@ -66,7 +66,7 @@ type CloudenvResourceListInput struct {
 	Providers []string `json:"providers"`
 	// swagger:ignore
 	// Deprecated
-	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
+	Provider []string `json:"provider" yunion-deprecated-by:"providers"`
 
 	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
 	//
@@ -77,7 +77,7 @@ type CloudenvResourceListInput struct {
 	Brands []string `json:"brands"`
 	// swagger:ignore
 	// Deprecated
-	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
+	Brand []string `json:"brand" yunion-deprecated-by:"brands"`
 
 	// 列出指定云环境的资源，支持云环境如下：
 	//
@@ -180,9 +180,9 @@ type CloudaccountCreateInput struct {
 	// 自动同步间隔时间
 	SyncIntervalSeconds int `json:"sync_interval_seconds"`
 
-	// 自动根据云上项目或订阅创建本地项目
+	// 自动根据云上项目或订阅创建本地项目, OpenStack此参数为true
 	// default: false
-	AutoCreateProject bool `json:"auto_create_project"`
+	AutoCreateProject *bool `json:"auto_create_project"`
 
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONDict `json:"options"`
@@ -227,7 +227,7 @@ type CloudaccountListInput struct {
 	// 代理
 	ProxySetting string `json:"proxy_setting"`
 	// swagger:ignore
-	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
+	ProxySettingId string `json:"proxy_setting_id" yunion-deprecated-by:"proxy_setting"`
 }
 
 type ProviderProject struct {

pkg/apis/compute/cloudprovider.go

@@ -147,15 +147,15 @@ type CloudproviderResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Manager string `json:"manager" "yunion:deprecated-by":"cloudprovider"`
+	Manager string `json:"manager" yunion-deprecated-by:"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	ManagerId string `json:"manager_id" "yunion:deprecated-by":"cloudprovider"`
+	ManagerId string `json:"manager_id" yunion-deprecated-by:"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudproviderId string `json:"cloudprovider_id" "yunion:deprecated-by":"cloudprovider"`
+	CloudproviderId string `json:"cloudprovider_id" yunion-deprecated-by:"cloudprovider"`
 }
 
 type ManagedResourceListInput struct {
@@ -169,15 +169,15 @@ type ManagedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudaccountId string `json:"cloudaccount_id" "yunion:deprecated-by":"cloudaccount"`
+	CloudaccountId string `json:"cloudaccount_id" yunion-deprecated-by:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Account string `json:"account" "yunion:deprecated-by":"cloudaccount"`
+	Account string `json:"account" yunion-deprecated-by:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	AccountId string `json:"account_id" "yunion:deprecated-by":"cloudaccount"`
+	AccountId string `json:"account_id" yunion-deprecated-by:"cloudaccount"`
 
 	// 过滤资源，是否为非OneCloud内置私有云管理的资源
 	// default: false
@@ -251,3 +251,23 @@ type CloudproviderUpdateInput struct {
 
 type CloudproviderCreateInput struct {
 }
+
+type CloudproviderGetStorageClassInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetStorageClassOutput struct {
+	// 对象存储存储类型
+	StorageClasses []string `json:"storage_classes"`
+}
+
+type CloudproviderGetCannedAclInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetCannedAclOutput struct {
+	// Bucket支持的预置ACL列表
+	BucketCannedAcls []string `json:"bucket_canned_acls"`
+	// Object支持的预置ACL列表
+	ObjectCannedAcls []string `json:"object_canned_acls"`
+}

pkg/apis/compute/cloudregions_const.go

@@ -51,7 +51,7 @@ const (
 	CITY_FU_ZHOU       = "Fuzhou"      //福州
 	CITY_WU_HAN        = "Wuhan"       //武汉
 	CITY_CHANG_SHA     = "Changsha"    //长沙
-	CITY_SHU_ZHOU      = "Shuzhou"     //苏州
+	CITY_SU_ZHOU       = "Suzhou"      //苏州
 	CITY_BAO_DING      = "Baoding"     //保定
 	CITY_NAN_JING      = "Nanjing"     //南京
 	CITY_FO_SHAN       = "Foshan"      //佛山

pkg/apis/compute/dbinstance.go

@@ -101,7 +101,7 @@ type DBInstanceCreateInput struct {
 	// | -----		| ------	| --- |
 	// | 华为云		|ha, single, replica| |
 	// | 阿里云		|basic, high_availability, always_on, finance||
-	// | Google		|SECOND_GEN | FIRST_GEN 目前谷歌已弃用|
+	// | Google		|Zonal, Regional | |
 	// 翻译:
 	// basic: 基础版
 	// high_availability: 高可用
@@ -110,7 +110,8 @@ type DBInstanceCreateInput struct {
 	// ha: 高可用
 	// single: 单机
 	// replica: 只读
-	// SECNOD_GEN: 第二代
+	// Zonal: 单区域
+	// Regional: 区域级
 	// required: true
 	Category string `json:"category"`
 
@@ -170,9 +171,22 @@ type SDBInstanceChangeConfigInput struct {
 type SDBInstanceRecoveryConfigInput struct {
 	apis.Meta
 
-	DBInstancebackup   string
-	DBInstancebackupId string            `json:"dbinstancebackup_id"`
-	Databases          map[string]string `json:"databases,allowempty"`
+	// swagger:ignore
+	DBInstancebackup string `json:"dbinstancebackup" yunion-deprecated-by:"dbinstancebackup_id"`
+
+	// 备份Id
+	//
+	//
+	// | 平台		| 支持引擎								| 说明		|
+	// | -----		| ------								| ---		|
+	// | 华为云		|MySQL, SQL Server						| 仅SQL Server支持恢复到当前实例			|
+	// | 阿里云		|MySQL, SQL Server						| MySQL要求必须开启单库单表恢复功能 并且只能是MySQL 8.0 高可用版（本地SSD盘）MySQL 5.7 高可用版（本地SSD盘）或MySQL 5.6 高可用版, MySQL仅支持恢复到当前实例|
+	// | Google		|MySQL, PostgreSQL, SQL Server			| PostgreSQL备份恢复时，要求实例不能有副本			|
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
+
+	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中, 阿里云此参数为必传
+	// example: {"sdb1":"ddb1"}
+	Databases map[string]string `json:"databases,allowempty"`
 }
 
 type DBInstanceListInput struct {
@@ -313,7 +327,7 @@ type DBInstanceResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	DBInstanceId string `json:"dbinstance_id" "yunion:deprecated-by":"dbinstance"`
+	DBInstanceId string `json:"dbinstance_id" yunion-deprecated-by:"dbinstance"`
 }
 
 type DBInstanceFilterListInputBase struct {

pkg/apis/compute/dbinstance_account.go

@@ -70,13 +70,13 @@ type SDBInstanceSetPrivilegesInput struct {
 
 type DBInstancePrivilege struct {
 	// 数据库名称
-	Database string
+	Database string `json:"database"`
 	// 账号名称
-	Account string
+	Account string `json:"account"`
 	// 数据库Id
-	DBInstancedatabaseId string
+	DBInstancedatabaseId string `json:"dbinstancedatabase_id"`
 	// 权限
-	Privileges string
+	Privileges string `json:"privileges"`
 }
 
 type DBInstanceAccountDetails struct {

pkg/apis/compute/dbinstance_const.go

@@ -16,19 +16,20 @@ package compute
 
 const (
 	//实例状态
-	DBINSTANCE_DEPLOYING      = "deploying"  //部署中
-	DBINSTANCE_RUNNING        = "running"    //运行中
-	DBINSTANCE_REBOOTING      = "rebooting"  //重启中
-	DBINSTANCE_MIGRATING      = "migrating"  //迁移中
-	DBINSTANCE_BACKING_UP     = "backing_up" //备份中
-	DBINSTANCE_RESTORING      = "restoring"  //备份恢复中
-	DBINSTANCE_RESTORE_FAILED = "restore_failed"
-	DBINSTANCE_IMPORTING      = "importing"     //数据导入中
-	DBINSTANCE_CLONING        = "cloning"       //克隆中
-	DBINSTANCE_DELETING       = "deleting"      //删除中
-	DBINSTANCE_DELETE_FAILED  = "delete_failed" //删除失败
-	DBINSTANCE_MAINTENANCE    = "maintenance"   //维护中
-	DBINSTANCE_UNKNOWN        = "unknown"
+	DBINSTANCE_DEPLOYING         = "deploying"         //部署中
+	DBINSTANCE_RUNNING           = "running"           //运行中
+	DBINSTANCE_REBOOTING         = "rebooting"         //重启中
+	DBINSTANCE_MIGRATING         = "migrating"         //迁移中
+	DBINSTANCE_BACKING_UP        = "backing_up"        //备份中
+	DBINSTANCE_BACKING_UP_FAILED = "backing_up_failed" //备份失败
+	DBINSTANCE_RESTORING         = "restoring"         //备份恢复中
+	DBINSTANCE_RESTORE_FAILED    = "restore_failed"
+	DBINSTANCE_IMPORTING         = "importing"     //数据导入中
+	DBINSTANCE_CLONING           = "cloning"       //克隆中
+	DBINSTANCE_DELETING          = "deleting"      //删除中
+	DBINSTANCE_DELETE_FAILED     = "delete_failed" //删除失败
+	DBINSTANCE_MAINTENANCE       = "maintenance"   //维护中
+	DBINSTANCE_UNKNOWN           = "unknown"
 
 	DBINSTANCE_CHANGE_CONFIG        = "change_config"        //调整配置
 	DBINSTANCE_CHANGE_CONFIG_FAILED = "change_config_failed" //调整配置失败
@@ -103,6 +104,10 @@ const (
 	HUAWEI_DBINSTANCE_CATEGORY_SINGLE  = "single"  //单机
 	HUAWEI_DBINSTANCE_CATEGORY_REPLICA = "replica" //只读
 
+	//谷歌云实例类型
+	GOOGLE_DBINSTANCE_CATEGORY_REGIONAL = "Regional" // 高可用性（区域级）
+	GOOGLE_DBINSTANCE_CATEGORY_ZONAL    = "Zonal"    // 单个地区
+
 	//阿里云存储类型
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD  = "local_ssd"  //本地盘SSD盘
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_CLOUD_ESSD = "cloud_essd" //ESSD云盘

pkg/apis/compute/disk.go

@@ -88,7 +88,7 @@ type SnapshotPolicyResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter disk by snapshotpolicy_id
-	SnapshotpolicyId string `json:"snapshotpolicy_id" "yunion:deprecated-by":"snapshotpolicy"`
+	SnapshotpolicyId string `json:"snapshotpolicy_id" yunion-deprecated-by:"snapshotpolicy"`
 }
 
 type SnapshotPolicyFilterListInput struct {
@@ -113,7 +113,7 @@ type DiskListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk type
-	Type string `json:"type" "yunion:deprecated-by":"disk_type"`
+	Type string `json:"type" yunion-deprecated-by:"disk_type"`
 	// 过滤指定disk_type的磁盘列表，可能的值为：sys, data, swap. volume
 	//
 	// | disk_type值 | 说明 |
@@ -137,13 +137,13 @@ type DiskListInput struct {
 	Template string `json:"template"`
 	// swagger:ignore
 	// Deprecated
-	TemplateId string `json:"template_id" "yunion:deprecated-by":"template"`
+	TemplateId string `json:"template_id" yunion-deprecated-by:"template"`
 
 	// 快照
 	Snapshot string `json:"snapshot"`
 	// swagger:ignore
 	// Deprecated
-	SnapshotId string `json:"snapshot_id" "yunion:deprecated-by":"snapshot"`
+	SnapshotId string `json:"snapshot_id" yunion-deprecated-by:"snapshot"`
 }
 
 type DiskResourceInput struct {
@@ -152,7 +152,7 @@ type DiskResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk_id
-	DiskId string `json:"disk_id" "yunion:deprecated-by":"disk"`
+	DiskId string `json:"disk_id" yunion-deprecated-by:"disk"`
 }
 
 type DiskFilterListInputBase struct {

pkg/apis/compute/dnsrecord.go

@@ -14,5 +14,11 @@
 
 package compute
 
+import "yunion.io/x/onecloud/pkg/apis"
+
+type DnsRecordCreateInput struct {
+	apis.AdminSharableVirtualResourceBaseCreateInput
+}
+
 type DnsRecordUpdateInput struct {
 }

pkg/apis/compute/elasticcache.go

@@ -53,7 +53,7 @@ type ELasticcacheResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ElasticcacheId string `json:"elasticcache_id" "yunion:deprecated-by":"elasticcache"`
+	ElasticcacheId string `json:"elasticcache_id" yunion-deprecated-by:"elasticcache"`
 }
 
 type ElasticcacheFilterListInput struct {

pkg/apis/compute/externalproject.go

@@ -16,9 +16,16 @@ package compute
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const (
+	EXTERNAL_PROJECT_STATUS_AVAILABLE   = "available"   // 可用
+	EXTERNAL_PROJECT_STATUS_UNAVAILABLE = "unavailable" // 不可用
+	EXTERNAL_PROJECT_STATUS_CREATING    = "creating"    // 创建中
+	EXTERNAL_PROJECT_STATUS_DELETING    = "deleting"    // 删除中
+	EXTERNAL_PROJECT_STATUS_UNKNOWN     = "unknown"     // 未知
+)
+
 type ExternalProjectDetails struct {
-	apis.StandaloneResourceDetails
-	apis.ProjectizedResourceInfo
+	apis.VirtualResourceDetails
 	ManagedResourceInfo
 
 	SExternalProject

pkg/apis/compute/geo_input.go

@@ -24,15 +24,15 @@ type CloudregionResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudregionId string `json:"cloudregion_id" "yunion:deprecated-by":"cloudregion"`
+	CloudregionId string `json:"cloudregion_id" yunion-deprecated-by:"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Region string `json:"region" "yunion:deprecated-by":"cloudregion"`
+	Region string `json:"region" yunion-deprecated-by:"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	RegionId string `json:"region_id" "yunion:deprecated-by":"cloudregion"`
+	RegionId string `json:"region_id" yunion-deprecated-by:"cloudregion"`
 }
 
 type RegionalFilterListInput struct {
@@ -64,6 +64,23 @@ type ZonalFilterListBase struct {
 	OrderByZone string `json:"order_by_zone"`
 }
 
+func (input ZonalFilterListBase) ZoneList() []string {
+	zoneStr := input.Zone
+	if len(zoneStr) > 0 {
+		input.Zones = append(input.Zones, zoneStr)
+	}
+	return input.Zones
+}
+
+func (input ZonalFilterListBase) FirstZone() string {
+	if len(input.Zone) > 0 {
+		return input.Zone
+	}
+	if len(input.Zones) > 0 {
+		return input.Zones[0]
+	}
+	return ""
+}
 func (input ZonalFilterListInput) ZoneList() []string {
 	zoneStr := input.Zone
 	if len(zoneStr) > 0 {
@@ -117,5 +134,5 @@ type ZoneResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ZoneId string `json:"zone_id" "yunion:deprecated-by":"zone"`
+	ZoneId string `json:"zone_id" yunion-deprecated-by:"zone"`
 }

pkg/apis/compute/globalvpc.go

@@ -42,7 +42,7 @@ type GlobalVpcResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	GlobalvpcId string `json:"globalvpc_id" "yunion:deprecated-by":"globalvpc"`
+	GlobalvpcId string `json:"globalvpc_id" yunion-deprecated-by:"globalvpc"`
 }
 
 type GlobalVpcResourceListInput struct {

pkg/apis/compute/guests.go

@@ -30,13 +30,13 @@ type ServerListInput struct {
 
 	HostFilterListInput
 
-	NetworkFilterListInput `"yunion:ambiguous-prefix":"vpc_"`
+	NetworkFilterListInput `yunion-ambiguous-prefix:"vpc_"`
 
 	billing.BillingResourceListInput
 
 	GroupFilterListInput
 	SecgroupFilterListInput
-	//DiskFilterListInput `"yunion:ambiguous-prefix":"storage_"`
+	//DiskFilterListInput `yunion-ambiguous-prefix:"storage_"`
 	ScalingGroupFilterListInput
 
 	// 只列出裸金属主机
@@ -66,13 +66,13 @@ type ServerListInput struct {
 	// 列出可以挂载磁盘的主机
 	AttachableServersForDisk string `json:"attachable_servers_for_disk"`
 	// Deprecated:列出可以挂载磁盘的主机
-	Disk string `json:"disk" "yunion:deprecated-by":"attachable_servers_for_disk"`
+	Disk string `json:"disk" yunion-deprecated-by:"attachable_servers_for_disk"`
 
 	// 按主机资源类型进行排序
 	// enum: shared,prepaid,dedicated
 	ResourceType string `json:"resource_type"`
-	// 返回开启主备机功能的主机
-	GetBackupGuestsOnHost *bool `json:"get_backup_guests_on_host"`
+	// 返回该宿主机上的所有虚拟机，包括备份机
+	GetAllGuestsOnHost string `json:"get_all_guests_on_host"`
 
 	// 根据宿主机 SN 过滤
 	// HostSn string `json:"host_sn"`
@@ -267,15 +267,15 @@ type ServerResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	ServerId string `json:"server_id" "yunion:deprecated-by":"server"`
+	ServerId string `json:"server_id" yunion-deprecated-by:"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	Guest string `json:"guest" "yunion:deprecated-by":"server"`
+	Guest string `json:"guest" yunion-deprecated-by:"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	GuestId string `json:"guest_id" "yunion:deprecated-by":"server"`
+	GuestId string `json:"guest_id" yunion-deprecated-by:"server"`
 }
 
 type ServerFilterListInput struct {

pkg/apis/compute/guesttemplate.go

@@ -57,6 +57,7 @@ type GuestTemplateDetails struct {
 
 	Secgroups []string `json:"secgroups"`
 	Zone      string   `json:"zone"`
+	ZoneId    string   `json:"zone_id"`
 	Brand     string   `json:"brand"`
 
 	ConfigInfo GuestTemplateConfigInfo `json:"config_info"`
@@ -71,9 +72,7 @@ type GuestTemplateListInput struct {
 }
 
 type GuestTemplateConfigInfo struct {
-	Region               string                 `json:"region"`
-	Zone                 string                 `json:"zone"`
-	Hypervisor           string                 `json:"hypervisor"`
+	Metadata             map[string]string      `json:"metadata"`
 	Secgroup             string                 `json:"secgroup"`
 	Sku                  GuestTemplateSku       `json:"sku"`
 	Disks                []GuestTemplateDisk    `json:"disks"`

pkg/apis/compute/host.go

@@ -138,7 +138,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率
@@ -164,7 +164,8 @@ type HostDetails struct {
 	CanPrepare        bool                `json:"can_prepare"`
 	PrepareFailReason string              `json:"prepare_fail_reason"`
 	// 允许开启宿主机健康检查
-	AllowHealthCheck bool `json:"allow_health_check"`
+	AllowHealthCheck      bool `json:"allow_health_check"`
+	AutoMigrateOnHostDown bool `json:"auto_migrate_on_host_down"`
 
 	// 标签
 	Metadata map[string]string `json:"metadata"`
@@ -223,7 +224,7 @@ type HostResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by host_id
-	HostId string `json:"host_id" "yunion:deprecated-by":"host"`
+	HostId string `json:"host_id" yunion-deprecated-by:"host"`
 }
 
 type HostRegisterMetadata struct {

pkg/apis/compute/hostjoin.go

@@ -21,7 +21,7 @@ type HostJointResourceDetailsBase struct {
 	Host string `json:"host"`
 	// 裸金属服务器名称
 	// Deprecated
-	Baremetal string `json:"baremetal" "yunion:deprecated-by":"host"`
+	Baremetal string `json:"baremetal" yunion-deprecated-by:"host"`
 }
 
 type HostJointResourceDetails struct {

pkg/apis/compute/input.go

@@ -66,8 +66,7 @@ type CachedimageListInput struct {
 }
 
 type ExternalProjectListInput struct {
-	apis.StandaloneResourceListInput
-	apis.ProjectizedResourceListInput
+	apis.VirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput
@@ -150,7 +149,7 @@ type GuestTemplateFilterListInput struct {
 	GuestTemplate string `json:"guest_template"`
 	// swagger:ignore
 	// Deprecated
-	GuestTemplateId string `json:"guest_template_id" "yunion:deprecated-by":"guest_template"`
+	GuestTemplateId string `json:"guest_template_id" yunion-deprecated-by:"guest_template"`
 }
 
 type ServiceCatalogListInput struct {

pkg/apis/compute/instance_group.go

@@ -50,7 +50,7 @@ type GroupResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by instance group Id
-	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
+	GroupId string `json:"group_id" yunion-deprecated-by:"group"`
 }
 
 type GroupFilterListInput struct {

pkg/apis/compute/loadbalancer.go

@@ -48,6 +48,8 @@ type LoadbalancerListenerListInput struct {
 
 	Scheduler []string `json:"scheduler"`
 
+	Certificate []string `json:"certificate_id"`
+
 	SendProxy []string `json:"send_proxy"`
 
 	AclStatus []string `json:"acl_status"`
@@ -168,6 +170,8 @@ type LoadbalancerDetails struct {
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
+	LoadbalancerClusterResourceInfo
+
 	VpcResourceInfoBase
 	ZoneResourceInfoBase
 	NetworkResourceInfoBase
@@ -197,7 +201,10 @@ type LoadbalancerResourceInfo struct {
 	// 可用区ID
 	ZoneId string `json:"zone_id"`
 
-	ZoneResourceInfoBase
+	ZoneResourceInfo
+
+	// cloud provider info
+	ManagedResourceInfo
 }
 
 type LoadbalancerResourceInput struct {
@@ -206,7 +213,7 @@ type LoadbalancerResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	LoadbalancerId string `json:"loadbalancer_id" "yunion:deprecated-by":"loadbalancer"`
+	LoadbalancerId string `json:"loadbalancer_id" yunion-deprecated-by:"loadbalancer"`
 }
 
 type LoadbalancerFilterListInput struct {

pkg/apis/compute/loadbalancerbackendgroup.go

@@ -39,7 +39,7 @@ type LoadbalancerBackendGroupResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	BackendGroupId string `json:"backend_group_id" "yunion:deprecated-by":"backend_group"`
+	BackendGroupId string `json:"backend_group_id" yunion-deprecated-by:"backend_group"`
 }
 
 type LoadbalancerBackendGroupFilterListInput struct {

pkg/apis/compute/loadbalancercluster.go

@@ -45,7 +45,7 @@ type LoadbalancerClusterResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ClusterId string `json:"cluster_id" "yunion:deprecated-by":"cluster"`
+	ClusterId string `json:"cluster_id" yunion-deprecated-by:"cluster"`
 }
 
 type LoadbalancerClusterFilterListInput struct {

pkg/apis/compute/loadbalancerlistener.go

@@ -45,7 +45,7 @@ type LoadbalancerListenerResourceInput struct {
 	// 负载均衡监听器ID
 	// swagger:ignore
 	// Deprecated
-	ListenerId string `json:"listener_id" "yunion:deprecated-by":"listener"`
+	ListenerId string `json:"listener_id" yunion-deprecated-by:"listener"`
 }
 
 type LoadbalancerListenerFilterListInput struct {

pkg/apis/compute/natgateway.go

@@ -89,7 +89,7 @@ type NatGatewayResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	NatgatewayId string `json:"natgateway_id" "yunion:deprecated-by":"natgateway"`
+	NatgatewayId string `json:"natgateway_id" yunion-deprecated-by:"natgateway"`
 }
 
 type NatGatewayFilterListInput struct {

pkg/apis/compute/network.go

@@ -24,7 +24,7 @@ type WireResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// fitler by wire id
-	WireId string `json:"wire_id" "yunion:deprecated-by":"wire"`
+	WireId string `json:"wire_id" yunion-deprecated-by:"wire"`
 }
 
 type WireFilterListBase struct {
@@ -47,7 +47,7 @@ type NetworkResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by networkId
-	NetworkId string `json:"network_id" "yunion:deprecated-by":"network"`
+	NetworkId string `json:"network_id" yunion-deprecated-by:"network"`
 }
 
 type NetworkFilterListBase struct {

pkg/apis/compute/scaling_group.go

@@ -134,9 +134,15 @@ type ScalingGroupDetails struct {
 	// example: OneCloud
 	Brand string `json:"brand"`
 
-	// description: 网络ID
-	// example: net-12345
-	Networks []string `json:"networks"`
+	// description: 网络信息
+	Networks []ScalingGroupNetwork `json:"networks"`
+}
+
+type ScalingGroupNetwork struct {
+	Id           string `json:"id"`
+	Name         string `json:"name"`
+	GuestIpStart string `json:"guest_ip_start"`
+	GuestIpEnd   string `json:"guest_ip_end"`
 }
 
 type ScalingGroupResourceInfo struct {

pkg/apis/compute/schedtag.go

@@ -45,7 +45,7 @@ type SchedtagResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by schedtag_id
-	SchedtagId string `json:"schedtag_id" "yunion:deprecated-by":"schedtag"`
+	SchedtagId string `json:"schedtag_id" yunion-deprecated-by:"schedtag"`
 }
 
 type SchedtagFilterListInput struct {
@@ -69,7 +69,7 @@ type SchedtagListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by type, alias for resource_type
-	Type string `json:"type" "yunion:deprecated-by":"resource_type"`
+	Type string `json:"type" yunion-deprecated-by:"resource_type"`
 
 	DefaultStrategy []string `json:"default_strategy"`
 }

pkg/apis/compute/secgroup.go

@@ -145,8 +145,18 @@ type SecgroupListInput struct {
 	// pattern:asc|desc
 	OrderByGuestCnt string `json:"order_by_guest_cnt"`
 
-	// 是否被修改
-	IsDirty *bool `json:"is_dirty"`
+	// 模糊过滤规则中含有指定ip的安全组
+	// example: 10.10.2.1
+	Ip string `json:"ip"`
+
+	// 精确匹配规则中含有指定端口的安全组
+	// example: 100-200
+	Ports string `json:"ports"`
+
+	// 指定过滤规则的方向(仅在指定ip或ports时生效) choices: all|in|out
+	// default: all
+	// example: in
+	Direction string `json:"direction"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -164,12 +174,18 @@ type SecurityGroupRuleListInput struct {
 	apis.ResourceBaseListInput
 	SecgroupFilterListInput
 
+	Projects []string `json:"projects"`
+
 	// 以direction字段过滤安全组规则
 	Direction string `json:"direction"`
 	// 以action字段过滤安全组规则
 	Action string `json:"action"`
 	// 以protocol字段过滤安全组规则
 	Protocol string `json:"protocol"`
+	// 以ports字段过滤安全组规则
+	Ports string `json:"ports"`
+	// 根据ip模糊匹配安全组规则
+	Ip string `json:"ip"`
 }
 
 type SecgroupResourceInput struct {
@@ -178,7 +194,10 @@ type SecgroupResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by secgroup_id
-	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
+	SecgroupId string `json:"secgroup_id" yunion-deprecated-by:"secgroup"`
+
+	// 模糊匹配安全组规则名称
+	SecgroupName string `json:"secgroup_name"`
 }
 
 type SecgroupFilterListInput struct {
@@ -193,15 +212,22 @@ type SecgroupDetails struct {
 	SSecurityGroup
 
 	// 关联云主机数量
-	GuestCnt int `json:"guest_cnt"`
+	GuestCnt int `json:"guest_cnt,allowempty"`
+
+	// 关联此安全组的云主机is_system为true数量
+	SystemGuestCnt int `json:"system_guest_cnt,allowempty"`
+
+	// admin_secgrp_id为此安全组的云主机数量
+	AdminGuestCnt int `json:"admin_guest_cnt,allowempty"`
+
 	// 安全组缓存数量
-	CacheCnt int `json:"cache_cnt"`
+	CacheCnt int `json:"cache_cnt,allowempty"`
 	// 规则信息
-	Rules string `json:"rules"`
+	Rules []SSecurityGroupRule `json:"rules"`
 	// 入方向规则信息
-	InRules string `json:"in_rules"`
+	InRules []SSecurityGroupRule `json:"in_rules"`
 	// 出方向规则信息
-	OutRules string `json:"out_rules"`
+	OutRules []SSecurityGroupRule `json:"out_rules"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/storage_const.go

@@ -81,6 +81,7 @@ const (
 	STORAGE_GOOGLE_LOCAL_SSD   = "local-ssd"   //本地SSD暂存盘 (最多8个)
 	STORAGE_GOOGLE_PD_STANDARD = "pd-standard" //标准永久性磁盘
 	STORAGE_GOOGLE_PD_SSD      = "pd-ssd"      //SSD永久性磁盘
+	STORAGE_GOOGLE_PD_BALANCED = "pd-balanced" //平衡永久性磁盘
 
 	// ctyun storage type
 	STORAGE_CTYUN_SSD  = "SSD"  // 超高IO云硬盘
@@ -127,7 +128,7 @@ var (
 		STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS,
 	}
 
-	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA}
 
 	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
 
@@ -144,7 +145,7 @@ type StorageResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by storage_id
-	StorageId string `json:"storage_id" "yunion:deprecated-by":"storage"`
+	StorageId string `json:"storage_id" yunion-deprecated-by:"storage"`
 }
 
 type StorageFilterListInputBase struct {

pkg/apis/compute/storagecachedimage.go

@@ -48,13 +48,13 @@ type StoragecachedimageListInput struct {
 	Storagecache string `json:"storagecache"`
 	// Deprecated
 	// swagger:ignore
-	StoragecacheId string `json:"storagecache_id" "yunion:deprecated-by":"storagecache"`
+	StoragecacheId string `json:"storagecache_id" yunion-deprecated-by:"storagecache"`
 
 	// 以镜像缓存过滤
 	Cachedimage string `json:"cachedimage"`
 	// Deprecated
 	// swagger:ignore
-	CachedimageId string `json:"cachedimage_id" "yunion:deprecated-by":"cachedimage"`
+	CachedimageId string `json:"cachedimage_id" yunion-deprecated-by:"cachedimage"`
 
 	// 镜像状态
 	Status []string `json:"status"`

pkg/apis/compute/vpc.go

@@ -80,7 +80,7 @@ type VpcResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by vpc Id
-	VpcId string `json:"vpc_id" "yunion:deprecated-by":"vpc"`
+	VpcId string `json:"vpc_id" yunion-deprecated-by:"vpc"`
 }
 
 type VpcFilterListInputBase struct {

pkg/apis/identity/config.go

@@ -28,6 +28,8 @@ type SLDAPIdpConfigBaseOptions struct {
 	Suffix   string `json:"suffix,omitempty" required:"true"`
 	User     string `json:"user,omitempty" required:"true"`
 	Password string `json:"password,omitempty" required:"true"`
+
+	DisableUserOnImport bool `json:"disable_user_on_import"`
 }
 
 type SLDAPIdpConfigSingleDomainOptions struct {
@@ -51,6 +53,8 @@ type SLDAPIdpConfigOptions struct {
 	User     string `json:"user,omitempty"`
 	Password string `json:"password,omitempty"`
 
+	DisableUserOnImport bool `json:"disable_user_on_import"`
+
 	DomainTreeDN        string `json:"domain_tree_dn,omitempty" help:"Domain tree root node dn(distinguished name)"`
 	DomainFilter        string `json:"domain_filter,omitempty"`
 	DomainObjectclass   string `json:"domain_objectclass,omitempty"`

pkg/apis/identity/identityprovider.go

@@ -73,7 +73,7 @@ type IdentityProviderCreateInput struct {
 	TargetDomain string `json:"target_domain"`
 	// swagger:ignore
 	// Deprecated
-	TargetDomainId string `json:"target_domain_id" "yunion:deprecated-by":"target_domain"`
+	TargetDomainId string `json:"target_domain_id" yunion-deprecated-by:"target_domain"`
 
 	// 新建域的时候是否自动新建第一个项目
 	AutoCreateProject *bool `json:"auto_create_project"`

pkg/apis/identity/input.go

@@ -49,22 +49,22 @@ type ProjectFilterListInput struct {
 	ProjectDomain string `json:"project_domain"`
 	// swagger:ignore
 	// Deprecated
-	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project-domain"`
+	ProjectDomainId string `json:"project_domain_id" yunion-deprecated-by:"project-domain"`
 
 	// 以项目（ID或Name）过滤列表结果
 	Project string `json:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by project_id
-	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
+	ProjectId string `json:"project_id" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant
-	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
+	Tenant string `json:"tenant" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant_id
-	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
+	TenantId string `json:"tenant_id" yunion-deprecated-by:"project"`
 }
 
 type UserFilterListInput struct {
@@ -79,7 +79,7 @@ type UserFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by user_id
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 }
 
 type GroupFilterListInput struct {
@@ -94,7 +94,7 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by group_id
-	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
+	GroupId string `json:"group_id" yunion-deprecated-by:"group"`
 }
 
 type RoleFilterListInput struct {
@@ -109,7 +109,7 @@ type RoleFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by role_id
-	RoleId string `json:"role_id" "yunion:deprecated-by":"role"`
+	RoleId string `json:"role_id" yunion-deprecated-by:"role"`
 }
 
 type ServiceFilterListInput struct {
@@ -121,7 +121,7 @@ type ServiceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by service_id
-	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
+	ServiceId string `json:"service_id" yunion-deprecated-by:"service"`
 
 	// 以服务名称排序
 	OrderByService string `json:"order_by_service"`
@@ -151,6 +151,9 @@ type ProjectListInput struct {
 
 	UserFilterListInput
 	GroupFilterListInput
+
+	// 过滤出指定用户或者组可以加入的项目
+	Jointable *bool `json:"jointable"`
 }
 
 type DomainListInput struct {
@@ -322,7 +325,7 @@ type RegionFilterListInput struct {
 	Region string `json:"region"`
 	// swagger:ignore
 	// Deprecated
-	RegionId string `json:"region_id" "yunion:deprecated-by":"region"`
+	RegionId string `json:"region_id" yunion-deprecated-by:"region"`
 }
 
 type RegionListInput struct {

pkg/apis/image/consts.go

@@ -48,6 +48,7 @@ const (
 	IMAGE_IS_READONLY         = "is_readonly"
 	IMAGE_PARTITION_TYPE      = "partition_type"
 	IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init"
+	IMAGE_DISABLE_USB_KBD     = "disable_usb_kbd"
 
 	IMAGE_STATUS_UPDATING = "updating"
 )

pkg/apis/image/image.go

@@ -88,3 +88,12 @@ type ImageCreateInput struct {
 	// 镜像属性
 	Properties map[string]string `json:"properties"`
 }
+
+type ImageUpdateStatusInput struct {
+	apis.Meta
+
+	// 镜像状态
+	Status string `json:"status"`
+	// 更新镜像状态原因
+	Reason string `json:"reason"`
+}

pkg/apis/input.go

@@ -21,15 +21,15 @@ type DomainizedResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	Domain string `json:"domain" "yunion:deprecated-by":"project_domain"`
+	Domain string `json:"domain" yunion-deprecated-by:"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Project domain Id filter, alias for project_domain
-	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project_domain"`
+	ProjectDomainId string `json:"project_domain_id" yunion-deprecated-by:"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Domain Id filter, alias for project_domain
-	DomainId string `json:"domain_id" "yunion:deprecated-by":"project_domain"`
+	DomainId string `json:"domain_id" yunion-deprecated-by:"project_domain"`
 }
 
 type ProjectizedResourceInput struct {
@@ -39,15 +39,15 @@ type ProjectizedResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by project_id, alias for project
-	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
+	ProjectId string `json:"project_id" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant ID or Name, alias for project
-	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
+	Tenant string `json:"tenant" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant_id, alias for project
-	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
+	TenantId string `json:"tenant_id" yunion-deprecated-by:"project"`
 }
 
 type DomainizedResourceCreateInput struct {
@@ -75,6 +75,27 @@ type SharableVirtualResourceCreateInput struct {
 	SharableResourceBaseCreateInput
 }
 
+type AdminSharableVirtualResourceBaseCreateInput struct {
+	SharableVirtualResourceCreateInput
+
+	// 记录
+	Records string `json:"records"`
+}
+
+type StatusDomainLevelUserResourceCreateInput struct {
+	StatusDomainLevelResourceCreateInput
+
+	// 本地用户Id，若为空则使用当前用户Id作为此参数值
+	OwnerId string `json:"owner_id"`
+}
+
+type UserResourceCreateInput struct {
+	StandaloneResourceCreateInput
+
+	// 本地用户Id，若为空则使用当前用户Id作为此参数值
+	OwnerId string `json:"owner_id"`
+}
+
 type VirtualResourceCreateInput struct {
 	StatusStandaloneResourceCreateInput
 	ProjectizedResourceCreateInput

pkg/apis/list.go

@@ -50,7 +50,7 @@ type ProjectizedResourceListInput struct {
 	OrderByProject string `json:"order_by_project"`
 	// swagger:ignore
 	// Deprecated
-	OrderByTenant string `json:"order_by_tenant" "yunion:deprecated-by":"order_by_project"`
+	OrderByTenant string `json:"order_by_tenant" yunion-deprecated-by:"order_by_project"`
 }
 
 type UserResourceListInput struct {
@@ -59,7 +59,7 @@ type UserResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by userId
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 }
 
 type ModelBaseListInput struct {

pkg/apis/output.go

@@ -137,11 +137,11 @@ type ProjectizedResourceInfo struct {
 
 	// 资源归属项目的ID(向后兼容别名）
 	// Deprecated
-	TenantId string `json:"project_id" "yunion:deprecated-by":"tenant_id"`
+	TenantId string `json:"project_id" yunion-deprecated-by:"tenant_id"`
 
 	// 资源归属项目的名称（向后兼容别名）
 	// Deprecated
-	Tenant string `json:"project" "yunion:deprecated-by":"tenant"`
+	Tenant string `json:"project" yunion-deprecated-by:"tenant"`
 }
 
 type ScopedResourceBaseInfo struct {

pkg/apis/yunionconf/input.go

@@ -28,14 +28,14 @@ type ParameterListInput struct {
 
 	// Deprecated
 	// swagger:ignore
-	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
+	ServiceId string `json:"service_id" yunion-deprecated-by:"service"`
 
 	// 用户名称或ID
 	User string `json:"user"`
 
 	// Deprecated
 	// swagger:ignore
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 
 	// filter by name
 	Name []string `json:"name"`

pkg/appsrv/ring.go

@@ -25,10 +25,12 @@ type Ring struct {
 }
 
 func NewRing(size int) *Ring {
-	r := Ring{buffer: make([]interface{}, size+1),
+	r := Ring{
+		buffer: make([]interface{}, size+1),
 		header: 0,
 		tail:   0,
-		lock:   &sync.Mutex{}}
+		lock:   &sync.Mutex{},
+	}
 	return &r
 }
 
@@ -58,6 +60,7 @@ func (r *Ring) Pop() interface{} {
 		return nil
 	}
 	ret := r.buffer[r.tail]
+	r.buffer[r.tail] = nil
 	r.tail = nextPointer(r.tail, len(r.buffer))
 	return ret
 }

pkg/appsrv/ring_test.go

@@ -19,28 +19,33 @@ import (
 )
 
 func TestRing(t *testing.T) {
-	r := NewRing(10)
-	var v int32 = 10
-	r.Push(v)
-	v = 20
-	r.Push(v)
-	v = 30
-	r.Push(v)
-	v1 := r.Pop().(int32)
-	if v1 != 10 {
-		t.Error("Fail")
-	}
-	v2 := r.Pop().(int32)
-	if v2 != 20 {
-		t.Error("Fail")
-	}
-	v3 := r.Pop().(int32)
-	if v3 != 30 {
-		t.Error("Fail")
-	}
-	v4 := r.Pop()
-	if v4 != nil {
-		t.Error("Fail")
+	var (
+		r    = NewRing(10)
+		push = func(v int32) {
+			r.Push(v)
+		}
+		pop = func(want int32) {
+			got := r.Pop().(int32)
+			if got != want {
+				t.Fatalf("got %d, want %d", got, want)
+			}
+			for i := r.header; i != r.tail; i = nextPointer(i, len(r.buffer)) {
+				if r.buffer[i] != nil {
+					t.Fatalf("head %d, tail %d, index %d not nil",
+						r.header, r.tail, i)
+				}
+			}
+		}
+	)
+	push(10)
+	push(20)
+	push(30)
+
+	pop(10)
+	pop(20)
+	pop(30)
+	if v := r.Pop(); v != nil {
+		t.Fatalf("want nil, got %#v", v)
 	}
 }
 

pkg/baremetal/pxe/dhcp.go

@@ -269,6 +269,9 @@ func (req *dhcpRequest) findNetworkConf(session *mcclient.ClientSession, filterU
 		params.Add(jsonutils.NewString(
 			fmt.Sprintf("guest_dhcp.endswith(',%s')", req.RelayAddr)),
 			"filter.3")
+		params.Add(jsonutils.NewString(
+			fmt.Sprintf("guest_dhcp.equals(%s)", req.RelayAddr)),
+			"filter.4")
 		params.Add(jsonutils.JSONTrue, "filter_any")
 	}
 	params.Add(jsonutils.JSONTrue, "is_on_premise")

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/app/auth.go

@@ -87,8 +87,9 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
-		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
-			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,
+		policy.EnableGlobalRbac(
+			time.Second*time.Duration(options.RbacPolicySyncPeriodSeconds),
+			time.Second*time.Duration(options.RbacPolicySyncFailedRetrySeconds),
 			options.RbacDebug,
 		)
 	}

pkg/cloudcommon/db/adminsharablevirtual.go

@@ -51,7 +51,22 @@ func (manager *SAdminSharableVirtualResourceBaseManager) GetIAdminSharableVirtua
 	return manager.GetVirtualObject().(IAdminSharableVirtualModelManager)
 }
 
-func (manager *SAdminSharableVirtualResourceBaseManager) ValidateCreateData(man IAdminSharableVirtualModelManager, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (manager *SAdminSharableVirtualResourceBaseManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input apis.AdminSharableVirtualResourceBaseCreateInput,
+) (apis.AdminSharableVirtualResourceBaseCreateInput, error) {
+	var err error
+	input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData")
+	}
+	return input, nil
+}
+
+func (manager *SAdminSharableVirtualResourceBaseManager) ValidateRecordsData(man IAdminSharableVirtualModelManager, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	records, err := man.ParseInputInfo(data)
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/domain.go

@@ -77,6 +77,10 @@ func (model *SDomainizedResourceBase) GetChangeOwnerCandidateDomainIds() []strin
 	return nil
 }
 
+func (model *SDomainizedResourceBase) GetChangeOwnerRequiredDomainIds() []string {
+	return nil
+}
+
 func ValidateCreateDomainId(domainId string) error {
 	if !consts.GetNonDefaultDomainProjects() && domainId != identity.DEFAULT_DOMAIN_ID {
 		return httperrors.NewForbiddenError("project in non-default domain is prohibited")

pkg/cloudcommon/db/domainresource.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -133,6 +134,9 @@ func (model *SDomainLevelResourceBase) AllowPerformChangeOwner(ctx context.Conte
 }
 
 func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformChangeDomainOwnerInput) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.GetIStandaloneModel().IsShared() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot change owner of shared resource")
 	}
@@ -162,6 +166,11 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 	if len(candidates) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), candidates) {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner candidate list")
 	}
+	requires := model.GetIDomainLevelModel().GetChangeOwnerRequiredDomainIds()
+	log.Debugf("%s required domains: %s", model.Keyword(), requires)
+	if len(requires) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), requires) {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner required list")
+	}
 
 	if !IsAdminAllowPerform(userCred, model, "change-owner") {
 		return nil, errors.Wrap(httperrors.ErrNotSufficientPrivilege, "require system privileges")

pkg/cloudcommon/db/external.go

@@ -66,7 +66,14 @@ func SetExternalId(model IExternalizedModel, userCred mcclient.TokenCredential,
 }
 
 func FetchByExternalId(manager IModelManager, idStr string) (IExternalizedModel, error) {
+	return FetchByExternalIdAndManagerId(manager, idStr, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		return q
+	})
+}
+
+func FetchByExternalIdAndManagerId(manager IModelManager, idStr string, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (IExternalizedModel, error) {
 	q := manager.Query().Equals("external_id", idStr)
+	q = filter(q)
 	count, err := q.CountWithError()
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/infraresource.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -200,6 +201,9 @@ func (model *SInfrasResourceBase) PerformChangeOwner(
 	query jsonutils.JSONObject,
 	input apis.PerformChangeDomainOwnerInput,
 ) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.IsShared() {
 		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot change owner when shared!")
 	}

pkg/cloudcommon/db/lockman/base.go

@@ -0,0 +1,51 @@
+package lockman
+
+import "context"
+
+type SBaseLockManager struct {
+	manager ILockManager
+}
+
+func NewBaseLockManger(m ILockManager) *SBaseLockManager {
+	return &SBaseLockManager{manager: m}
+}
+
+func (m *SBaseLockManager) LockClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.UnlockKey(ctx, key)
+}

pkg/cloudcommon/db/lockman/etcd.go

@@ -167,6 +167,7 @@ type SLockTableIndex struct {
 }
 
 type SEtcdLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[SLockTableIndex]*SEtcdLockRecord
 
@@ -203,6 +204,7 @@ func NewEtcdLockManager(config *SEtcdLockManagerConfig) (ILockManager, error) {
 		Value:  lockman,
 		Func:   atexit.ExitHandlerFunc(lockman.destroyAtExit),
 	})
+	lockman.SBaseLockManager = NewBaseLockManger(&lockman)
 	return &lockman, nil
 }
 

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/db/lockman/inmemory.go

@@ -135,12 +135,17 @@ func (rec *SInMemoryLockRecord) unlockContext(ctx context.Context) (needClean bo
 }
 
 type SInMemoryLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[string]*SInMemoryLockRecord
 }
 
 func NewInMemoryLockManager() ILockManager {
-	lockMan := SInMemoryLockManager{tableLock: &sync.Mutex{}, lockTable: make(map[string]*SInMemoryLockRecord)}
+	lockMan := SInMemoryLockManager{
+		tableLock: &sync.Mutex{},
+		lockTable: make(map[string]*SInMemoryLockRecord),
+	}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }
 

pkg/cloudcommon/db/lockman/lockman.go

@@ -31,6 +31,15 @@ type ILockedObject interface {
 type ILockManager interface {
 	LockKey(ctx context.Context, key string)
 	UnlockKey(ctx context.Context, key string)
+
+	LockClass(ctx context.Context, manager ILockedClass, projectId string)
+	ReleaseClass(ctx context.Context, manager ILockedClass, projectId string)
+	LockObject(ctx context.Context, model ILockedObject)
+	ReleaseObject(ctx context.Context, model ILockedObject)
+	LockRawObject(ctx context.Context, resName string, resId string)
+	ReleaseRawObject(ctx context.Context, resName string, resId string)
+	LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
+	ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
 }
 
 func getClassKey(manager ILockedClass, projectId string) string {
@@ -64,41 +73,33 @@ func Init(man ILockManager) {
 }
 
 func LockClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockClass(ctx, manager, projectId)
 }
 
 func ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseClass(ctx, manager, projectId)
 }
 
 func LockObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockObject(ctx, model)
 }
 
 func ReleaseObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseObject(ctx, model)
 }
 
 func LockRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockRawObject(ctx, resName, resId)
 }
 
 func ReleaseRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseRawObject(ctx, resName, resId)
 }
 
 func LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockJointObject(ctx, model, model2)
 }
 
 func ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseJointObject(ctx, model, model2)
 }

pkg/cloudcommon/db/lockman/noop.go

@@ -21,6 +21,7 @@ import (
 )
 
 type SNoopLockManager struct {
+	*SBaseLockManager
 }
 
 func (lockman *SNoopLockManager) LockKey(ctx context.Context, key string) {
@@ -33,5 +34,6 @@ func (lockman *SNoopLockManager) UnlockKey(ctx context.Context, key string) {
 
 func NewNoopLockManager() ILockManager {
 	lockMan := SNoopLockManager{}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }

pkg/cloudcommon/db/managed.go

@@ -22,6 +22,7 @@ import (
 
 type IOwnerResourceBaseModel interface {
 	GetChangeOwnerCandidateDomainIds() []string
+	GetChangeOwnerRequiredDomainIds() []string
 }
 
 type IManagedResourceBase interface {

pkg/cloudcommon/db/opslog.go

@@ -16,6 +16,7 @@ package db
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strconv"
 	"strings"
@@ -460,28 +461,7 @@ func (manager *SOpsLogManager) ListItemFilter(
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
 ) (*sqlchemy.SQuery, error) {
-	/*userStrs := jsonutils.GetQueryStringArray(query, "user")
-	if len(userStrs) > 0 {
-		for i := range userStrs {
-			usrObj, err := DefaultUserFetcher(ctx, userStrs[i])
-			if err != nil {
-				if err == sql.ErrNoRows {
-					return nil, httperrors.NewResourceNotFoundError2("user", userStrs[i])
-				} else if err == sqlchemy.ErrDuplicateEntry {
-					return nil, httperrors.NewDuplicateNameError("user", userStrs[i])
-				} else {
-					return nil, httperrors.NewGeneralError(err)
-				}
-			}
-			userStrs[i] = usrObj.GetId()
-		}
-		if len(userStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("user_id"), userStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("user_id"), userStrs))
-		}
-	}
-	projStrs := jsonutils.GetQueryStringArray(query, "project")
+	projStrs := jsonutils.GetQueryStringArray(query, "owner_project_ids")
 	if len(projStrs) > 0 {
 		for i := range projStrs {
 			projObj, err := DefaultProjectFetcher(ctx, projStrs[i])
@@ -494,12 +474,23 @@ func (manager *SOpsLogManager) ListItemFilter(
 			}
 			projStrs[i] = projObj.GetId()
 		}
-		if len(projStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("owner_tenant_id"), projStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+		q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+	}
+	domainStrs := jsonutils.GetQueryStringArray(query, "owner_domain_ids")
+	if len(domainStrs) > 0 {
+		for i := range domainStrs {
+			domainObj, err := DefaultDomainFetcher(ctx, domainStrs[i])
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("domain", domainStrs[i])
+				} else {
+					return nil, httperrors.NewGeneralError(err)
+				}
+			}
+			domainStrs[i] = domainObj.GetId()
 		}
-	}*/
+		q = q.Filter(sqlchemy.In(q.Field("owner_domain_id"), domainStrs))
+	}
 	objTypes := jsonutils.GetQueryStringArray(query, "obj_type")
 	if len(objTypes) > 0 {
 		if len(objTypes) == 1 {
@@ -621,17 +612,11 @@ func (self *SOpsLogManager) FilterByOwner(q *sqlchemy.SQuery, ownerId mcclient.I
 			}
 		case rbacutils.ScopeProject:
 			if len(ownerId.GetProjectId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_tenant_id"), ownerId.GetProjectId()),
-					sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()))
 			}
 		case rbacutils.ScopeDomain:
 			if len(ownerId.GetProjectDomainId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_domain_id"), ownerId.GetProjectDomainId()),
-					sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()))
 			}
 		default:
 			// systemScope, no filter

pkg/cloudcommon/db/quotas/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -481,7 +482,7 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) {
@@ -524,7 +525,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
@@ -616,3 +617,33 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 	}
 	return ret, nil
 }
+
+type tQuotaResultList []jsonutils.JSONObject
+
+func (a tQuotaResultList) Len() int           { return len(a) }
+func (a tQuotaResultList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a tQuotaResultList) Less(i, j int) bool { return usageRateOfQuota(a[i]) > usageRateOfQuota(a[j]) }
+
+func usageRateOfQuota(quota jsonutils.JSONObject) float32 {
+	maxRate := float32(0)
+	quotaMap, _ := quota.GetMap()
+	for k, v := range quotaMap {
+		usageK := fmt.Sprintf("usage.%s", k)
+		if usageV, ok := quotaMap[usageK]; ok {
+			intV, _ := v.Int()
+			if intV > 0 {
+				intUsageV, _ := usageV.Int()
+				rate := float32(intUsageV) / float32(intV)
+				if maxRate < rate {
+					maxRate = rate
+				}
+			}
+		}
+	}
+	return maxRate
+}
+
+func sortQuotaByUsage(quotaList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	sort.Sort(tQuotaResultList(quotaList))
+	return quotaList
+}

pkg/cloudcommon/db/quotas/interface.go

@@ -71,6 +71,7 @@ type IQuotaManager interface {
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
 	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)

pkg/cloudcommon/db/quotas/quotas.go

@@ -113,6 +113,17 @@ func (manager *SQuotaBaseManager) _cancelUsage(ctx context.Context, userCred mcc
 	return manager.changeUsage(ctx, userCred, usage, false)
 }
 
+func (manager *SQuotaBaseManager) addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	LockQuota(ctx, manager, usage)
+	defer ReleaseQuota(ctx, manager, usage)
+
+	return manager._addUsage(ctx, userCred, usage)
+}
+
+func (manager *SQuotaBaseManager) _addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	return manager.changeUsage(ctx, userCred, usage, true)
+}
+
 func (manager *SQuotaBaseManager) changeUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota, isAdd bool) error {
 	usages, err := manager.usageStore.GetParentQuotas(ctx, usage.GetKeys())
 	if err != nil {

pkg/cloudcommon/db/quotas/register.go

@@ -87,6 +87,20 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 	}
 }
 
+func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	for _, usage := range usages {
+		addUsage(ctx, userCred, usage.(IQuota))
+	}
+}
+
+func addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) {
+	manager := getQuotaManager(usage)
+	err := manager.addUsage(ctx, userCred, usage)
+	if err != nil {
+		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
+	}
+}
+
 func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
 	manager := getQuotaManager(request)
 	return manager.getQuotaCount(ctx, request, pendingKeys)

pkg/cloudcommon/db/sharablebase.go

@@ -162,8 +162,14 @@ func SharableManagerValidateCreateData(
 			input.IsPublic = &isPublic
 			reqScope = rbacutils.ScopeSystem
 		} else if input.PublicScope == string(rbacutils.ScopeDomain) {
-			input.IsPublic = &isPublic
-			reqScope = rbacutils.ScopeDomain
+			if consts.GetNonDefaultDomainProjects() {
+				// only if non_default_domain_projects turned on, allow sharing to domain
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeDomain
+			} else {
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeSystem
+			}
 		} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
 			// backward compatible, if only is_public is true, make it share to system
 			input.IsPublic = &isPublic
@@ -171,17 +177,23 @@ func SharableManagerValidateCreateData(
 			reqScope = rbacutils.ScopeSystem
 		} else {
 			input.IsPublic = nil
-			input.PublicScope = string(rbacutils.ScopeNone)
+			input.PublicScope = "" // string(rbacutils.ScopeNone)
 		}
 	case rbacutils.ScopeDomain:
-		if input.PublicScope == string(rbacutils.ScopeSystem) {
-			input.IsPublic = &isPublic
-			reqScope = rbacutils.ScopeSystem
-		} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
-			// backward compatible, if only is_public is true, make it share to system
-			input.IsPublic = &isPublic
-			input.PublicScope = string(rbacutils.ScopeSystem)
-			reqScope = rbacutils.ScopeSystem
+		if consts.GetNonDefaultDomainProjects() {
+			// only if non_default_domain_projects turned on, allow sharing domain resources
+			if input.PublicScope == string(rbacutils.ScopeSystem) {
+				input.IsPublic = &isPublic
+				reqScope = rbacutils.ScopeSystem
+			} else if input.IsPublic != nil && *input.IsPublic && len(input.PublicScope) == 0 {
+				// backward compatible, if only is_public is true, make it share to system
+				input.IsPublic = &isPublic
+				input.PublicScope = string(rbacutils.ScopeSystem)
+				reqScope = rbacutils.ScopeSystem
+			} else {
+				input.IsPublic = nil
+				input.PublicScope = "" // string(rbacutils.ScopeNone)
+			}
 		} else {
 			input.IsPublic = nil
 			input.PublicScope = string(rbacutils.ScopeNone)
@@ -423,6 +435,9 @@ func SharablePerformPublic(model ISharableBaseModel, ctx context.Context, userCr
 			targetScope = rbacutils.ScopeNone
 		}
 	case rbacutils.ScopeDomain:
+		if !consts.GetNonDefaultDomainProjects() {
+			return errors.Wrap(httperrors.ErrForbidden, "not allow to share to domain when non_default_domain_projects turned off")
+		}
 		if len(requireIds) == 0 {
 			return errors.Wrap(httperrors.ErrForbidden, "require to be shared to system")
 		}
@@ -555,7 +570,8 @@ func SharableModelIsShared(model ISharableBaseModel) bool {
 func SharableModelCustomizeCreate(model ISharableBaseModel, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	if !data.Contains("public_scope") {
 		resScope := model.GetModelManager().ResourceScope()
-		if resScope == rbacutils.ScopeDomain {
+		if resScope == rbacutils.ScopeDomain && consts.GetNonDefaultDomainProjects() {
+			// only if non_default_domain_projects turned on, do the following
 			isManaged := false
 			if managedModel, ok := model.(IManagedResourceBase); ok {
 				isManaged = managedModel.IsManaged()

pkg/cloudcommon/db/standalone.go

@@ -488,6 +488,10 @@ func (manager *SStandaloneResourceBaseManager) ValidateCreateData(ctx context.Co
 	if err != nil {
 		return input, errors.Wrap(err, "SResourceBaseManager.ValidateCreateData")
 	}
+	input.Name = strings.TrimSpace(input.Name)
+	if strings.ContainsAny(input.Name, "\n\r\t") {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "name should not contains any \\n\\r\\t")
+	}
 	return input, nil
 }
 
@@ -578,6 +582,10 @@ func (model *SStandaloneResourceBase) ValidateUpdateData(ctx context.Context, us
 	}
 
 	if len(input.Name) > 0 {
+		input.Name = strings.TrimSpace(input.Name)
+		if strings.ContainsAny(input.Name, "\n\r\t") {
+			return input, errors.Wrap(httperrors.ErrInputParameter, "name should not contains any \\n\\r\\t")
+		}
 		err = alterNameValidator(model.GetIStandaloneModel(), input.Name)
 		if err != nil {
 			return input, errors.Wrap(err, "alterNameValidator")

pkg/cloudcommon/notifyclient/notify.go

@@ -126,7 +126,7 @@ func RawNotify(recipientId []string, isGroup bool, channel notify.TNotifyChannel
 	msg.Topic = topic
 	body, _ := getContent(event, "content", channel, data)
 	if len(body) == 0 {
-		body = data.String()
+		body, _ = data.GetString()
 	}
 	msg.Msg = body
 	// log.Debugf("send notification %s %s", topic, body)