Merge pull request #7548 from wanyaoqi/automated-cherry-pick-of-#7547-upstream-release-3.3

Automated cherry pick of #7547: support diable usb kbd

build/docker/Dockerfile.ansibleserver

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:1.0.0
 
-ADD ./_output/bin/ansibleserver /opt/yunion/bin/ansibleserver
+ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -1,13 +1,12 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
+FROM alpine:3.11.6
 
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk update && \
-    apk add --no-cache tzdata git ansible ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+# openssh-client, for ansible ssh connection
+# git, ca-certificates, for fetching ansible roles
+RUN set -x \
+	&& apk update \
+	&& apk add openssh-client \
+	&& apk add ansible \
+	&& apk add tzdata git ca-certificates \
+	&& rm -rf /var/cache/apk/*

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudid

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudid /opt/yunion/bin/cloudid

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/logger /opt/yunion/bin/logger
 

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/monitor /opt/yunion/bin/monitor
 

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -7,7 +7,7 @@ ENV TZ Asia/Shanghai
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates && \
+    apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.yunionconf

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.1
 
 ADD ./_output/bin/yunionconf /opt/yunion/bin/yunionconf
 

cmd/climc/shell/cloudid/cloudgroup.go

@@ -28,6 +28,7 @@ func init() {
 
 		ClouduserId   string `json:"clouduser_id"`
 		CloudpolicyId string `json:"cloudpolicy_id"`
+		Usable        bool   `json:"usable"`
 	}
 	R(&CloudgroupListOptions{}, "cloud-group-list", "List cloud groups", func(s *mcclient.ClientSession, opts *CloudgroupListOptions) error {
 		params, err := options.ListStructToParams(opts)

cmd/climc/shell/compute/globalvpcs.go

@@ -15,6 +15,8 @@
 package compute
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -78,4 +80,30 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	R(&GlobalVpcShowOptions{}, "global-vpc-change-owner-candidate-domains", "Show candiate domains of a global vpc for changing owner", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
+		result, err := modules.GlobalVpcs.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type GlobalVpcChangeOwnerOptions struct {
+		ID            string `help:"ID or name of vpc" json:"-"`
+		ProjectDomain string `json:"project_domain" help:"target domain"`
+	}
+	R(&GlobalVpcChangeOwnerOptions{}, "global-vpc-change-owner", "Change owner domain of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcChangeOwnerOptions) error {
+		if len(args.ProjectDomain) == 0 {
+			return fmt.Errorf("empty project_domain")
+		}
+		params := jsonutils.Marshal(args)
+		ret, err := modules.GlobalVpcs.PerformAction(s, args.ID, "change-owner", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/compute/guest_template.go

@@ -145,6 +145,15 @@ func init() {
 		},
 	)
 
+	R(&GuestTemplateOptions{}, "server-template-inspect", "Inspect server template", func(s *mcclient.ClientSession, opts *GuestTemplateOptions) error {
+		tem, err := modules.GuestTemplate.PerformAction(s, opts.ID, "inspect", jsonutils.JSONNull)
+		if err != nil {
+			return err
+		}
+		printObject(tem)
+		return nil
+	})
+
 	type GuestTemplatePublicOptions struct {
 		ID          string `help:"ID or Name of server template"`
 		PublicScope string `help:"public scope"`

cmd/climc/shell/compute/schedulers.go

@@ -111,10 +111,11 @@ func init() {
 	type SchedulerCleanCacheOptions struct {
 		HostId    string `help:"ID of host" short-token:"h"`
 		SessionId string `help:"Session id" short-token:"s"`
+		Sync      bool   `help:"Sync do clean sched cache"`
 	}
 	R(&SchedulerCleanCacheOptions{}, "scheduler-clean-cache", "Clean scheduler hosts cache",
 		func(s *mcclient.ClientSession, args *SchedulerCleanCacheOptions) error {
-			err := modules.SchedManager.CleanCache(s, args.HostId, args.SessionId)
+			err := modules.SchedManager.CleanCache(s, args.HostId, args.SessionId, args.Sync)
 			if err != nil {
 				return err
 			}

cmd/climc/shell/compute/secgrouprules.go

@@ -25,33 +25,20 @@ import (
 func init() {
 	type SecGroupRulesListOptions struct {
 		options.BaseListOptions
-		Secgroup  string `help:"Secgroup ID or Name"`
-		Direction string `help:"filter Direction of rule" choices:"in|out"`
-		Protocol  string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Action    string `help:"filter Actin of rule" choices:"allow|deny"`
+		Secgroup     string   `help:"Secgroup ID or Name"`
+		SecgroupName string   `help:"Search rules by fuzzy secgroup name"`
+		Projects     []string `help:"Filter rules by project"`
+		Direction    string   `help:"filter Direction of rule" choices:"in|out"`
+		Protocol     string   `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
+		Action       string   `help:"filter Actin of rule" choices:"allow|deny"`
+		Ports        string   `help:"filter Ports of rule"`
+		Ip           string   `help:"filter cidr of rule"`
 	}
 
 	R(&SecGroupRulesListOptions{}, "secgroup-rule-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupRulesListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Secgroup) > 0 {
-			params.Add(jsonutils.NewString(args.Secgroup), "secgroup")
-		}
-		if len(args.Direction) > 0 {
-			params.Add(jsonutils.NewString(args.Direction), "direction")
-		}
-		if len(args.Protocol) > 0 {
-			params.Add(jsonutils.NewString(args.Protocol), "protocol")
-		}
-		if len(args.Action) > 0 {
-			params.Add(jsonutils.NewString(args.Action), "action")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroupRules.List(s, params)
 		if err != nil {

cmd/climc/shell/compute/secgroups.go

@@ -29,23 +29,15 @@ func init() {
 		Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"`
 		Server string `help:"Filter secgroups bound to specified server"`
 		options.BaseListOptions
+		Ip        string `help:"Filter secgroup by ip"`
+		Ports     string `help:"Filter secgroup by ports"`
+		Direction string `help:"Filter secgroup by ports" choices:"all|in|out"`
 	}
 
 	R(&SecGroupsListOptions{}, "secgroup-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupsListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-		}
-		if len(args.Equals) > 0 {
-			params.Add(jsonutils.NewString(args.Equals), "equals")
-		}
-		if len(args.Server) > 0 {
-			params.Add(jsonutils.NewString(args.Server), "server")
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
 		result, err := modules.SecGroups.List(s, params)
 		if err != nil {

cmd/climc/shell/compute/servers.go

@@ -298,6 +298,12 @@ func init() {
 		return nil
 	})
 
+	R(&options.ServerIdsOptions{}, "server-reconcile-backup", "Reconcile backup server", func(s *mcclient.ClientSession, opts *options.ServerIdsOptions) error {
+		ret := modules.Servers.BatchPerformAction(s, opts.ID, "reconcile-backup", nil)
+		printBatchResults(ret, modules.Servers.GetColumns(s))
+		return nil
+	})
+
 	R(&options.ServerIdsOptions{}, "server-create-backup", "Create backup guest", func(s *mcclient.ClientSession, opts *options.ServerIdsOptions) error {
 		ret := modules.Servers.BatchPerformAction(s, opts.ID, "create-backup", nil)
 		printBatchResults(ret, modules.Servers.GetColumns(s))
@@ -1146,6 +1152,7 @@ func init() {
 		ID               string `help:"ID or name of VM"`
 		DisableIsaSerial string `help:"disable isa serial device" choices:"true|false"`
 		DisablePvpanic   string `help:"disable pvpanic device" choices:"true|false"`
+		DisableUsbKbd    string `help:"disable usb kbd" choices:"true|false"`
 	}
 
 	R(&ServerQemuParams{}, "server-set-qemu-params", "config qemu params", func(s *mcclient.ClientSession,
@@ -1157,6 +1164,9 @@ func init() {
 		if len(opts.DisablePvpanic) > 0 {
 			params.Set("disable_pvpanic", jsonutils.NewString(opts.DisablePvpanic))
 		}
+		if len(opts.DisableUsbKbd) > 0 {
+			params.Set("disable_usb_kbd", jsonutils.NewString(opts.DisableUsbKbd))
+		}
 		result, err := modules.Servers.PerformAction(s, opts.ID, "set-qemu-params", params)
 		if err != nil {
 			return err

cmd/climc/shell/image/images.go

@@ -54,6 +54,7 @@ type ImageOptionalOptions struct {
 	Hypervisor         []string `help:"Prefer hypervisor type" choices:"kvm|esxi|baremetal|container|openstack|ctyun"`
 	DiskDriver         string   `help:"Perfer disk driver" choices:"virtio|scsi|pvscsi|ide|sata"`
 	NetDriver          string   `help:"Preferred network driver" choices:"virtio|e1000|vmxnet3"`
+	DisableUsbKbd      bool     `help:"Disable usb keyboard on this image(for hypervisor kvm)"`
 }
 
 func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDict, args ImageOptionalOptions) error {
@@ -131,6 +132,9 @@ func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDi
 	if len(args.Hypervisor) > 0 {
 		params.Add(jsonutils.NewString(strings.Join(args.Hypervisor, ",")), "properties", "hypervisor")
 	}
+	if args.DisableUsbKbd {
+		params.Add(jsonutils.NewString("true"), "properties", "disable_usb_kbd")
+	}
 	return nil
 }
 
@@ -252,7 +256,7 @@ func init() {
 	}
 
 	type ImageDeleteOptions struct {
-		ID                    []string `help:"Image ID or name"`
+		ID                    []string `help:"Image ID or name" json:"-"`
 		OverridePendingDelete *bool    `help:"Delete image directly instead of pending delete" short-token:"f"`
 	}
 	R(&ImageDeleteOptions{}, "image-delete", "Delete a image", func(s *mcclient.ClientSession, args *ImageDeleteOptions) error {

cmd/climc/shell/k8s/clusters.go

@@ -188,6 +188,19 @@ func initKubeCluster() {
 		return nil
 	})
 
+	R(&o.ClusterSyncOptions{}, cmdN("sync"), "Sync cluster k8s resources", func(s *mcclient.ClientSession, args *o.ClusterSyncOptions) error {
+		param, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "sync", param)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
 	R(&o.IdentOptions{}, cmdN("public"), "Make cluster public", func(s *mcclient.ClientSession, args *o.IdentOptions) error {
 		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "public", nil)
 		if err != nil {

cmd/climc/shell/k8s/k8s.go

@@ -43,7 +43,6 @@ func init() {
 	initRepo()
 	initChart()
 	initRelease()
-	initReleaseApps()
 
 	// kubernetes original resources
 	initRaw()

cmd/climc/shell/k8s/list.go

@@ -30,7 +30,7 @@ import (
 )
 
 func getPrinterRowValue(printer k8s.ListPrinter, obj jsonutils.JSONObject, col string) interface{} {
-	getFuncName := fmt.Sprintf("Get%s", strings.Title(col))
+	getFuncName := fmt.Sprintf("Get_%s", strings.Title(col))
 	manValue := reflect.ValueOf(printer)
 	funcValue := manValue.MethodByName(getFuncName)
 	if !funcValue.IsValid() || funcValue.IsNil() {

cmd/climc/shell/k8s/rbac.go

@@ -22,8 +22,10 @@ import (
 
 func initRbac() {
 	//cmdRole := initK8sNamespaceResource("rbacrole", k8s.RbacRoles)
-	initK8sNamespaceResource("rbacrole", k8s.RbacRoles)
-	initK8sNamespaceResource("rbacrolebinding", k8s.RbacRoleBindings)
+	initK8sNamespaceResource("rbac-role", k8s.RbacRoles)
+	initK8sNamespaceResource("rbac-rolebinding", k8s.RbacRoleBindings)
+	initK8sClusterResource("rbac-clusterrole", k8s.RbacClusterRoles)
+	initK8sClusterResource("rbac-clusterrolebinding", k8s.RbacClusterRoleBindings)
 	initK8sNamespaceResource("serviceaccount", k8s.ServiceAccounts)
 	//cmdRoleN := cmdRole.CommandNameFactory
 }

cmd/climc/shell/k8s/releaseapps.go

@@ -1,67 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package k8s
-
-import (
-	"fmt"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
-	o "yunion.io/x/onecloud/pkg/mcclient/options/k8s"
-)
-
-func initReleaseApps() {
-	cmdN := func(app string) string {
-		return resourceCmdN(fmt.Sprintf("app-%s", app), "create")
-	}
-	R(&o.AppCreateOptions{}, cmdN("meter"), "Create yunion meter helm release", func(s *mcclient.ClientSession, args *o.AppCreateOptions) error {
-		params, err := args.Params()
-		if err != nil {
-			return err
-		}
-		ret, err := k8s.MeterReleaseApps.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(ret)
-		return nil
-	})
-
-	R(&o.AppCreateOptions{}, cmdN("servicetree"), "Create yunion servicetree helm release", func(s *mcclient.ClientSession, args *o.AppCreateOptions) error {
-		params, err := args.Params()
-		if err != nil {
-			return err
-		}
-		ret, err := k8s.ServicetreeReleaseApps.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(ret)
-		return nil
-	})
-
-	R(&o.AppCreateOptions{}, cmdN("notify"), "Create yunion notify helm release", func(s *mcclient.ClientSession, args *o.AppCreateOptions) error {
-		params, err := args.Params()
-		if err != nil {
-			return err
-		}
-		ret, err := k8s.NotifyReleaseApps.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(ret)
-		return nil
-	})
-}

cmd/climc/shell/misc/copyright.go

@@ -35,6 +35,7 @@ func init() {
 		Copyright string  `help:"The copyright"`
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
+		License   *string `help:"the license generator website address"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -47,6 +48,10 @@ func init() {
 			params.Add(jsonutils.NewString(*args.Docs), "docs")
 		}
 
+		if args.License != nil {
+			params.Add(jsonutils.NewString(*args.License), "license")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

cmd/climc/shell/monitor/commonalert.go

@@ -0,0 +1,47 @@
+package monitor
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type CommonAlertListOptions struct {
+		options.BaseListOptions
+		// 报警类型
+		AlertType string `help:"common alert type" choices:"normal|system"`
+		Level     string `help:"common alert notify level" choices:"normal|important|fatal"`
+	}
+	R(&CommonAlertListOptions{}, "commonalert-list", "List commonalert", func(s *mcclient.ClientSession,
+		args *CommonAlertListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.CommonAlertManager.List(s, params)
+		if err != nil {
+			return nil
+		}
+		printList(result, modules.CommonAlertManager.GetColumns(s))
+		return nil
+	})
+
+	type CommonAlertDeleteOptions struct {
+		ID    string `help:"ID of alart"`
+		Force bool   `help:"force to delete alert"`
+	}
+	R(&CommonAlertDeleteOptions{}, "commonalert-delete", "List commonalert", func(s *mcclient.ClientSession,
+		args *CommonAlertDeleteOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewBool(args.Force), "force")
+		object, err := modules.CommonAlertManager.Delete(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(object)
+		return nil
+	})
+}

cmd/googlecli/main.go

@@ -18,17 +18,21 @@ import (
 	"fmt"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
 	Debug        bool   `help:"debug mode"`
 	Help         bool   `help:"Show help"`
+	AuthFile     string `help:"google cloud auth json file path" default:"$GOOGLE_AUTH_FILE"`
 	ClientEmail  string `help:"Client email" default:"$GOOGLE_CLIENT_EMAIL"`
 	ProjectID    string `help:"Project ID" default:"$GOOGLE_PROJECT_ID"`
 	PrivateKeyID string `help:"Private Key ID" default:"$GOOGLE_PRIVATE_KEY_ID"`
@@ -78,6 +82,20 @@ func showErrorAndExit(e error) {
 }
 
 func newClient(options *BaseOptions) (*google.SRegion, error) {
+	if len(options.AuthFile) > 0 {
+		jsonStr, err := fileutils2.FileGetContents(options.AuthFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "FileGetContents")
+		}
+		jsonCfg, err := jsonutils.ParseString(jsonStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.ParseString")
+		}
+		options.ClientEmail, _ = jsonCfg.GetString("client_email")
+		options.PrivateKeyID, _ = jsonCfg.GetString("private_key_id")
+		options.PrivateKey, _ = jsonCfg.GetString("private_key")
+		options.ProjectID, _ = jsonCfg.GetString("project_id")
+	}
 	if len(options.ClientEmail) == 0 {
 		return nil, fmt.Errorf("Missing ClientEmail")
 	}

go.mod

@@ -131,13 +131,13 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200615014624-f9c3576579c9
+	yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20200615071345-60a252beb982
+	yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200608080702-9b6683aa048c
-	yunion.io/x/structarg v0.0.0-20200619020316-f93fbe32ffc7
+	yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11
+	yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8
 )
 
 replace (

go.sum

@@ -94,7 +94,9 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/a8m/mark v0.1.1-0.20170507133748-44f2db618845/go.mod h1:c8Mh99Cw82nrsAnPgxQSZHkswVOJF7/MqZb1ZdvriLM=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20180112035716-b5829a0278c8 h1:mb7rn6WX8viizO36Nv6VVCBrgwaq01HPrZv1qq07/Yg=
 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20180112035716-b5829a0278c8/go.mod h1:T9M45xf79ahXVelWoOBmH0y4aC1t5kXO5BxwyakgIGA=
@@ -1016,6 +1018,7 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
@@ -1123,10 +1126,9 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200615014624-f9c3576579c9 h1:s7sobXxn4ckRHdjIaA91Rnfy2kreV/+dN3P2pQLg7F0=
-yunion.io/x/jsonutils v0.0.0-20200615014624-f9c3576579c9/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383 h1:XEhTD+FGjBjEIWwcY0Ez1lY3oTTzjUOcRgH/i4eFM5A=
+yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383/go.mod h1:3U5hspYn9apQHaXKY+fJ4Qgwjey32Kc+Mq+mUe9NamQ=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
-yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
@@ -1134,12 +1136,13 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 h1:gjqNa2uQzIEXyySsbvVz0Ult
 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200615071345-60a252beb982 h1:VnE92PZ2DOBn7qOKHHHPrwo4tPsjrxET/xDyWfMFQHI=
-yunion.io/x/pkg v0.0.0-20200615071345-60a252beb982/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b h1:wwPzH8iMRYjtaSM2pBN7nOqeZxR4XW5wdz2VE1KH/Ug=
+yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b h1:Ov/eKwwNeQje7SggkopExzcCW/GfaUouvXkcY6RUEcM=
+yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200608080702-9b6683aa048c h1:/i93mWSKLYUq8RsaXGbMpwwnkGi09wL/HdCBiexL/+c=
-yunion.io/x/sqlchemy v0.0.0-20200608080702-9b6683aa048c/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20200619020316-f93fbe32ffc7 h1:HjWH0gG/5KzvCxItnzRxizfAnMOvGcxPfsEOFwOGBhQ=
-yunion.io/x/structarg v0.0.0-20200619020316-f93fbe32ffc7/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11 h1:srrssomGJLh4OaDaY/1so+2FUBjZxdwQx/1pZ6403YU=
+yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8 h1:35Q2ce/gI2cuVhsuLS7WsL2K9SxwMEtWTopt/99iYsk=
+yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/service/handlers.go

@@ -39,6 +39,7 @@ func InitHandlers(app *appsrv.Application) {
 	db.InitAllManagers()
 
 	db.RegisterModelManager(db.OpsLog)
+	db.RegisterModelManager(db.Metadata)
 	db.RegisterModelManager(db.UserCacheManager)
 	db.RegisterModelManager(db.TenantCacheManager)
 	for _, manager := range []db.IModelManager{

pkg/apigateway/handler/auth.go

@@ -230,6 +230,9 @@ func doTenantLogin(ctx context.Context, w http.ResponseWriter, req *http.Request
 		} else {
 			otpVerified = true
 		}
+	} else {
+		// if totp disabled, then assume totp been verified
+		otpVerified = true
 	}
 
 	token, e = auth.Client().SetProject(tenantId, "", "", token)
@@ -396,7 +399,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
+		httperrors.ForbiddenError(w, "user forbidden login from web")
 		return false
 	}
 	return true
@@ -536,6 +539,11 @@ func (h *AuthHandlers) postLoginHandler(ctx context.Context, w http.ResponseWrit
 			httperrors.GeneralServerError(w, err)
 			return
 		}
+	} else {
+		// if totp is disabled, assume totp been verified
+		totp := clientman.TokenMan.GetTotp(tid)
+		totp.MarkVerified()
+		clientman.TokenMan.SaveTotp(tid)
 	}
 
 	appsrv.Send(w, qrcode)

pkg/apigateway/handler/imageutils.go

@@ -0,0 +1,129 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func readImageForm(r *multipart.Reader) (map[string]string, *multipart.Part, error) {
+	params := make(map[string]string)
+
+	maxValueBytes := int64(10 << 20)
+	for {
+		p, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+		filename := p.FileName()
+
+		var b bytes.Buffer
+
+		_, hasContentTypeHeader := p.Header["Content-Type"]
+		if !hasContentTypeHeader && filename == "" {
+			// value, store as string in memory
+			n, err := io.CopyN(&b, p, maxValueBytes+1)
+			if err != nil && err != io.EOF {
+				return nil, nil, err
+			}
+			maxValueBytes -= n
+			if maxValueBytes < 0 {
+				return nil, nil, multipart.ErrMessageTooLarge
+			}
+			params[name] = b.String()
+			continue
+		}
+
+		if name == "image" || name == "file" {
+			return params, p, nil
+		} else {
+			return nil, nil, fmt.Errorf("no file uploaded")
+		}
+	}
+	return nil, nil, fmt.Errorf("empty form")
+}
+
+func imageUploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	params := jsonutils.NewDict()
+
+	name, ok := p["name"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少镜像名称")
+		return
+	}
+	params.Add(jsonutils.NewString(name), "name")
+
+	_imageSize, ok := p["image_size"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少文件信息")
+		return
+	}
+	imageSize, e := strconv.ParseInt(_imageSize, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "文件信息错误")
+		return
+	}
+
+	// add all other params
+	for k, v := range p {
+		if k == "name" || k == "image_size" {
+			continue
+		}
+		params.Add(jsonutils.NewString(v), k)
+	}
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	res, e := modules.Images.Upload(s, params, f, imageSize)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	} else {
+		appsrv.SendJSON(w, res)
+	}
+}
+
+func uploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
+	log.Debugf("%s - %s", method, prefix)
+	hi := appsrv.SHandlerInfo{}
+	hi.SetMethod(method)
+	hi.SetPath(prefix)
+	hi.SetHandler(handler)
+	hi.SetProcessTimeout(6 * time.Hour)
+	hi.SetWorkerManager(GetUploaderWorker())
+	return &hi
+}

pkg/apigateway/handler/middleware.go

@@ -112,15 +112,15 @@ func FetchAuthToken(f func(context.Context, http.ResponseWriter, *http.Request))
 			return
 		}
 		// 启用双因子认证
-		t := AppContextToken(ctx)
-		if isUserEnableTotp(ctx, r, t) {
-			tid := getAuthToken(r)
-			totp := clientman.TokenMan.GetTotp(tid)
-			if !totp.IsVerified() {
-				httperrors.UnauthorizedError(w, "TOTP authentication failed")
-				return
-			}
+		// t := AppContextToken(ctx)
+		// if isUserEnableTotp(ctx, r, t) {
+		tid := getAuthToken(r)
+		totp := clientman.TokenMan.GetTotp(tid)
+		if !totp.IsVerified() {
+			httperrors.UnauthorizedError(w, "TOTP authentication failed")
+			return
 		}
+		// }
 
 		f(ctx, w, r)
 	}

pkg/apigateway/handler/misc.go

@@ -20,6 +20,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +39,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -57,6 +60,12 @@ const (
 	BATCH_HOST_REGISTER_QUANTITY_LIMITATION = 1000
 )
 
+var (
+	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
+	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+)
+
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
 	token := AppContextToken(ctx)
 	session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion)
@@ -82,6 +91,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	app.AddHandler3(uploader)
 	app.AddHandler(GET, prefix+"downloads/<template_id>", FetchAuthToken(h.getDownloadsHandler))
 	app.AddHandler(POST, prefix+"piuploads", FetchAuthToken(h.postPIUploads)) // itsm process instances upload api
+	imageUploader := uploadHandlerInfo("POST", prefix+"/imageutils/upload", FetchAuthToken(imageUploadHandler))
+	app.AddHandler3(imageUploader)
+	s3upload := uploadHandlerInfo(POST, prefix+"s3uploads", FetchAuthToken(h.postS3UploadHandler))
+	app.AddHandler3(s3upload)
 }
 
 func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
@@ -170,20 +183,43 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	// check header line
+	titlesOk := false
+	for _, t := range [][]string{BatchHostRegisterTemplate, BatchHostISORegisterTemplate, BatchHostPXERegisterTemplate} {
+		if len(t) == len(rows[0]) {
+			for _, title := range rows[0] {
+				if !utils.IsInStringArray(title, t) {
+					break
+				}
+			}
+
+			titlesOk = true
+		}
+	}
+
+	if !titlesOk {
+		httperrors.InputParameterError(w, "template file is invalid.please check.")
+		return
+	}
+
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +235,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 
@@ -297,18 +358,6 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			return
 		}
 
-		if _, ok := names[name]; ok {
-			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
-			httperrors.JsonClientError(w, e)
-			return
-		} else {
-			names[name] = true
-			_, err := modules.UsersV3.Get(s, name, nil)
-			if err == nil {
-				continue
-			}
-		}
-
 		domainId, ok := domains[domain]
 		if !ok {
 			if len(domain) == 0 {
@@ -327,6 +376,18 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			domains[domain] = id
 		}
 
+		if _, ok := names[name+"/"+domainId]; ok {
+			e := httperrors.NewClientError("row %d duplicate name %s", rowIdx, name)
+			httperrors.JsonClientError(w, e)
+			return
+		} else {
+			names[name+"/"+domainId] = true
+			_, err := modules.UsersV3.Get(s, name, nil)
+			if err == nil {
+				continue
+			}
+		}
+
 		user := jsonutils.NewDict()
 		user.Add(jsonutils.NewString(name), "name")
 		user.Add(jsonutils.NewString(domainId), "domain_id")
@@ -375,21 +436,21 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 	var content bytes.Buffer
 	switch template {
 	case "BatchHostRegister":
-		records := [][]string{{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}}
+		records := [][]string{BatchHostRegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostISORegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}}
+		records := [][]string{BatchHostISORegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostPXERegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}}
+		records := [][]string{BatchHostPXERegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
@@ -453,6 +514,61 @@ func (mh *MiscHandler) postPIUploads(ctx context.Context, w http.ResponseWriter,
 	appsrv.SendJSON(w, resp)
 }
 
+func (mh *MiscHandler) postS3UploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		log.Debugf("postS3UploadHandler.MultipartReader %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		log.Debugf("postS3UploadHandler.readImageForm %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	bucket_id, ok := p["bucket_id"]
+	if !ok {
+		httperrors.MissingParameterError(w, "bucket_id")
+		return
+	}
+
+	key, ok := p["key"]
+	if !ok {
+		httperrors.MissingParameterError(w, "key")
+		return
+	}
+
+	_content_length, ok := p["content_length"]
+	if !ok {
+		httperrors.MissingParameterError(w, "content_length")
+		return
+	}
+
+	content_length, e := strconv.ParseInt(_content_length, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "invalid content_length %s", _content_length)
+		return
+	}
+
+	storage_class, _ := p["storage_class"]
+	acl, _ := p["acl"]
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	meta := http.Header{}
+	meta.Set("Content-Type", "application/octet-stream")
+	e = modules.Buckets.Upload(s, bucket_id, key, f, content_length, storage_class, acl, meta)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.NewDict())
+}
+
 func writeCsv(records [][]string) (bytes.Buffer, error) {
 	var content bytes.Buffer
 	content.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM, 防止office打开后中文乱码

pkg/apis/cloudid/cloudgroup.go

@@ -80,6 +80,9 @@ type CloudgroupListInput struct {
 
 	// 根据权限过滤权限组
 	CloudpolicyId string `json:"cloudpolicy_id"`
+
+	// 是否可用
+	Usable *bool `json:"usable"`
 }
 
 type SCloudIdBaseResource struct {
@@ -105,15 +108,7 @@ type CloudgroupDetails struct {
 type CloudgroupCreateInput struct {
 	apis.StatusInfrasResourceBaseCreateInput
 
-	// 平台
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 支持                                        |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 支持                                        |
-	// | Qcloud   | 支持                                        |
+	// 平台, 目前支持Qcloud,Google,Azure,Aliyun,Huawei,Aws
 	Provider string `json:"provider"`
 
 	// 权限Id列表, 权限provider必须和权限组provider一致
@@ -123,84 +118,36 @@ type CloudgroupCreateInput struct {
 type CloudgroupAddUserInput struct {
 
 	// 用户Id
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 不支持                                      |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 支持                                        |
-	// | 腾讯云   | 支持                                        |
 	ClouduserId string `json:"clouduser_id"`
 }
 
 type CloudgroupRemoveUserInput struct {
 
 	// 用户Id
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 不支持                                      |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 支持                                        |
-	// | 腾讯云   | 支持                                        |
 	ClouduserId string `json:"clouduser_id"`
 }
 
 type CloudgroupAttachPolicyInput struct {
 
 	// 权限Id
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 不支持                                      |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 不支持                                      |
-	// | 腾讯云   | 支持                                        |
 	CloudpolicyId string `json:"cloudpolicy_id"`
 }
 
 type CloudgroupSetUsersInput struct {
 
 	// 公有云子账号Ids
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 支持                                        |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 支持                                        |
-	// | 腾讯云   | 支持                                        |
 	ClouduserIds []string `json:"clouduser_ids"`
 }
 
 type CloudgroupSetPoliciesInput struct {
 
 	// 权限Ids
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 支持                                        |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 不支持                                      |
-	// | 腾讯云   | 支持                                        |
 	CloudpolicyIds []string `json:"cloudpolicy_ids"`
 }
 
 type CloudgroupDetachPolicyInput struct {
 
 	// 权限Id
-	//
-	// | 云平台   | 说明                                        |
-	// |----------|---------------------------------------------|
-	// | Google   | 不支持                                      |
-	// | Aliyun   | 支持										|
-	// | Huawei   | 支持                                        |
-	// | Azure    | 不支持                                      |
-	// | 腾讯云   | 支持                                        |
 	CloudpolicyId string `json:"cloudpolicy_id"`
 }
 

pkg/apis/cloudid/cloudid.go

@@ -0,0 +1,22 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudid
+
+import "yunion.io/x/onecloud/pkg/apis"
+
+const (
+	SERVICE_TYPE    = apis.SERVICE_TYPE_CLOUDID
+	SERVICE_VERSION = ""
+)

pkg/apis/cloudid/clouduser.go

@@ -35,7 +35,7 @@ const (
 )
 
 type ClouduserCreateInput struct {
-	apis.StatusUserResourceCreateInput
+	apis.StatusDomainLevelUserResourceCreateInput
 	apis.StatusBaseResourceCreateInput
 
 	// 云订阅ID
@@ -88,7 +88,7 @@ type ClouduserCreateInput struct {
 	// | Google   | 至少需要一个初始权限                        |
 	// | Aliyun   | 支持										|
 	// | Huawei   | 不支持                                      |
-	// | Azure    | 不支持                                      |
+	// | Azure    | 支持                                        |
 	// | 腾讯云   | 支持                                        |
 	CloudpolicyIds []string `json:"cloudpolicy_ids"`
 
@@ -100,7 +100,7 @@ type ClouduserCreateInput struct {
 }
 
 type ClouduserListInput struct {
-	apis.StatusUserResourceListInput
+	apis.StatusDomainLevelUserResourceListInput
 
 	CloudaccountResourceListInput
 	CloudproviderResourceListInput
@@ -113,7 +113,7 @@ type ClouduserListInput struct {
 }
 
 type ClouduserDetails struct {
-	apis.StatusUserResourceDetails
+	apis.StatusDomainLevelUserResourceDetails
 	SClouduser
 
 	CloudaccountResourceDetails

pkg/apis/cloudid/zz_generated.model.go

@@ -22,10 +22,9 @@ import (
 type SCloudaccount struct {
 	apis.SStandaloneResourceBase
 	apis.SDomainizedResourceBase
-	Provider         string `json:"provider"`
-	Brand            string `json:"brand"`
-	IamLoginUrl      string `json:"iam_login_url"`
-	IsSupportCloudId *bool  `json:"is_support_cloud_id,omitempty"`
+	Provider    string `json:"provider"`
+	Brand       string `json:"brand"`
+	IamLoginUrl string `json:"iam_login_url"`
 }
 
 // SCloudaccountResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudid/models.SCloudaccountResourceBase.
@@ -103,7 +102,7 @@ type SCloudproviderResourceBase struct {
 
 // SClouduser is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudid/models.SClouduser.
 type SClouduser struct {
-	apis.SStatusUserResourceBase
+	apis.SStatusDomainLevelUserResourceBase
 	apis.SExternalizedResourceBase
 	SCloudproviderResourceBase
 	SCloudaccountResourceBase

pkg/apis/compute/api.go

@@ -289,7 +289,7 @@ type ServerConfigs struct {
 
 	// 虚拟机高可用(创建备机)
 	// default: false
-	// requried: false
+	// required: false
 	Backup bool `json:"backup"`
 
 	// 创建虚拟机数量
@@ -471,7 +471,8 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	OsType string `json:"os_type"`
-
+	// swagger:ignore
+	DisableUsbKbd bool `json:"disable_usb_kbd"`
 	// swagger:ignore
 	OsProfile jsonutils.JSONObject `json:"__os_profile__"`
 	// swagger:ignore

pkg/apis/compute/cachedloadbalanceracl.go

@@ -30,24 +30,3 @@ type CachedLoadbalancerAclListInput struct {
 	ManagedResourceListInput
 	LoadbalancerAclFilterListInput
 }
-
-type LoadbalancerAclResourceInfo struct {
-	// 负载均衡ACL名称
-	Acl string `json:"acl"`
-}
-
-type LoadbalancerAclResourceInput struct {
-	// ACL名称或ID
-	Acl string `json:"acl"`
-
-	// swagger:ignore
-	// Deprecated
-	AclId string `json:"acl_id" "yunion:deprecated-by":"acl"`
-}
-
-type LoadbalancerAclFilterListInput struct {
-	LoadbalancerAclResourceInput
-
-	// 以ACL名称排序
-	OrderByAcl string `json:"order_by_acl"`
-}

pkg/apis/compute/cachedloadbalancercertificate.go

@@ -31,24 +31,3 @@ type CachedLoadbalancerCertificateListInput struct {
 	RegionalFilterListInput
 	LoadbalancerCertificateFilterListInput
 }
-
-type LoadbalancerCertificateResourceInfo struct {
-	// 负载均衡证书名称
-	Certificate string `json:"certificate"`
-}
-
-type LoadbalancerCertificateResourceInput struct {
-	// 证书名称或ID
-	Certificate string `json:"certificate"`
-
-	// swagger:ignore
-	// Deprecated
-	CertificateId string `json:"certificate_id" "yunion:deprecated-by":"certificate"`
-}
-
-type LoadbalancerCertificateFilterListInput struct {
-	LoadbalancerCertificateResourceInput
-
-	// 以证书名称排序
-	OrderByCertificate string `json:"order_by_certificate"`
-}

pkg/apis/compute/cloudaccount.go

@@ -180,9 +180,9 @@ type CloudaccountCreateInput struct {
 	// 自动同步间隔时间
 	SyncIntervalSeconds int `json:"sync_interval_seconds"`
 
-	// 自动根据云上项目或订阅创建本地项目
+	// 自动根据云上项目或订阅创建本地项目, OpenStack此参数为true
 	// default: false
-	AutoCreateProject bool `json:"auto_create_project"`
+	AutoCreateProject *bool `json:"auto_create_project"`
 
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONDict `json:"options"`

pkg/apis/compute/cloudprovider.go

@@ -251,3 +251,23 @@ type CloudproviderUpdateInput struct {
 
 type CloudproviderCreateInput struct {
 }
+
+type CloudproviderGetStorageClassInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetStorageClassOutput struct {
+	// 对象存储存储类型
+	StorageClasses []string `json:"storage_classes"`
+}
+
+type CloudproviderGetCannedAclInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetCannedAclOutput struct {
+	// Bucket支持的预置ACL列表
+	BucketCannedAcls []string `json:"bucket_canned_acls"`
+	// Object支持的预置ACL列表
+	ObjectCannedAcls []string `json:"object_canned_acls"`
+}

pkg/apis/compute/dbinstance.go

@@ -109,7 +109,7 @@ type DBInstanceCreateInput struct {
 	// | -----		| ------	| --- |
 	// | 华为云		|ha, single, replica| |
 	// | 阿里云		|basic, high_availability, always_on, finance||
-	// | Google		|SECOND_GEN | FIRST_GEN 目前谷歌已弃用|
+	// | Google		|Zonal, Regional | |
 	// 翻译:
 	// basic: 基础版
 	// high_availability: 高可用
@@ -118,7 +118,8 @@ type DBInstanceCreateInput struct {
 	// ha: 高可用
 	// single: 单机
 	// replica: 只读
-	// SECNOD_GEN: 第二代
+	// Zonal: 单区域
+	// Regional: 区域级
 	// required: true
 	Category string `json:"category"`
 
@@ -178,9 +179,22 @@ type SDBInstanceChangeConfigInput struct {
 type SDBInstanceRecoveryConfigInput struct {
 	apis.Meta
 
-	DBInstancebackup   string
-	DBInstancebackupId string            `json:"dbinstancebackup_id"`
-	Databases          map[string]string `json:"databases,allowempty"`
+	// swagger:ignore
+	DBInstancebackup string `json:"dbinstancebackup" "yunion:deprecated-by":"dbinstancebackup_id"`
+
+	// 备份Id
+	//
+	//
+	// | 平台		| 支持引擎								| 说明		|
+	// | -----		| ------								| ---		|
+	// | 华为云		|MySQL, SQL Server						| 仅SQL Server支持恢复到当前实例			|
+	// | 阿里云		|MySQL, SQL Server						| MySQL要求必须开启单库单表恢复功能 并且只能是MySQL 8.0 高可用版（本地SSD盘）MySQL 5.7 高可用版（本地SSD盘）或MySQL 5.6 高可用版, MySQL仅支持恢复到当前实例|
+	// | Google		|MySQL, PostgreSQL, SQL Server			| PostgreSQL备份恢复时，要求实例不能有副本			|
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
+
+	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中, 阿里云此参数为必传
+	// example: {"sdb1":"ddb1"}
+	Databases map[string]string `json:"databases,allowempty"`
 }
 
 type DBInstanceListInput struct {

pkg/apis/compute/dbinstance_account.go

@@ -70,13 +70,13 @@ type SDBInstanceSetPrivilegesInput struct {
 
 type DBInstancePrivilege struct {
 	// 数据库名称
-	Database string
+	Database string `json:"database"`
 	// 账号名称
-	Account string
+	Account string `json:"account"`
 	// 数据库Id
-	DBInstancedatabaseId string
+	DBInstancedatabaseId string `json:"dbinstancedatabase_id"`
 	// 权限
-	Privileges string
+	Privileges string `json:"privileges"`
 }
 
 type DBInstanceAccountDetails struct {

pkg/apis/compute/dbinstance_const.go

@@ -16,19 +16,20 @@ package compute
 
 const (
 	//实例状态
-	DBINSTANCE_DEPLOYING      = "deploying"  //部署中
-	DBINSTANCE_RUNNING        = "running"    //运行中
-	DBINSTANCE_REBOOTING      = "rebooting"  //重启中
-	DBINSTANCE_MIGRATING      = "migrating"  //迁移中
-	DBINSTANCE_BACKING_UP     = "backing_up" //备份中
-	DBINSTANCE_RESTORING      = "restoring"  //备份恢复中
-	DBINSTANCE_RESTORE_FAILED = "restore_failed"
-	DBINSTANCE_IMPORTING      = "importing"     //数据导入中
-	DBINSTANCE_CLONING        = "cloning"       //克隆中
-	DBINSTANCE_DELETING       = "deleting"      //删除中
-	DBINSTANCE_DELETE_FAILED  = "delete_failed" //删除失败
-	DBINSTANCE_MAINTENANCE    = "maintenance"   //维护中
-	DBINSTANCE_UNKNOWN        = "unknown"
+	DBINSTANCE_DEPLOYING         = "deploying"         //部署中
+	DBINSTANCE_RUNNING           = "running"           //运行中
+	DBINSTANCE_REBOOTING         = "rebooting"         //重启中
+	DBINSTANCE_MIGRATING         = "migrating"         //迁移中
+	DBINSTANCE_BACKING_UP        = "backing_up"        //备份中
+	DBINSTANCE_BACKING_UP_FAILED = "backing_up_failed" //备份失败
+	DBINSTANCE_RESTORING         = "restoring"         //备份恢复中
+	DBINSTANCE_RESTORE_FAILED    = "restore_failed"
+	DBINSTANCE_IMPORTING         = "importing"     //数据导入中
+	DBINSTANCE_CLONING           = "cloning"       //克隆中
+	DBINSTANCE_DELETING          = "deleting"      //删除中
+	DBINSTANCE_DELETE_FAILED     = "delete_failed" //删除失败
+	DBINSTANCE_MAINTENANCE       = "maintenance"   //维护中
+	DBINSTANCE_UNKNOWN           = "unknown"
 
 	DBINSTANCE_CHANGE_CONFIG        = "change_config"        //调整配置
 	DBINSTANCE_CHANGE_CONFIG_FAILED = "change_config_failed" //调整配置失败
@@ -103,6 +104,10 @@ const (
 	HUAWEI_DBINSTANCE_CATEGORY_SINGLE  = "single"  //单机
 	HUAWEI_DBINSTANCE_CATEGORY_REPLICA = "replica" //只读
 
+	//谷歌云实例类型
+	GOOGLE_DBINSTANCE_CATEGORY_REGIONAL = "Regional" // 高可用性（区域级）
+	GOOGLE_DBINSTANCE_CATEGORY_ZONAL    = "Zonal"    // 单个地区
+
 	//阿里云存储类型
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD  = "local_ssd"  //本地盘SSD盘
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_CLOUD_ESSD = "cloud_essd" //ESSD云盘

pkg/apis/compute/externalproject.go

@@ -16,9 +16,16 @@ package compute
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const (
+	EXTERNAL_PROJECT_STATUS_AVAILABLE   = "available"   // 可用
+	EXTERNAL_PROJECT_STATUS_UNAVAILABLE = "unavailable" // 不可用
+	EXTERNAL_PROJECT_STATUS_CREATING    = "creating"    // 创建中
+	EXTERNAL_PROJECT_STATUS_DELETING    = "deleting"    // 删除中
+	EXTERNAL_PROJECT_STATUS_UNKNOWN     = "unknown"     // 未知
+)
+
 type ExternalProjectDetails struct {
-	apis.StandaloneResourceDetails
-	apis.ProjectizedResourceInfo
+	apis.VirtualResourceDetails
 	ManagedResourceInfo
 
 	SExternalProject

pkg/apis/compute/geo_input.go

@@ -64,6 +64,23 @@ type ZonalFilterListBase struct {
 	OrderByZone string `json:"order_by_zone"`
 }
 
+func (input ZonalFilterListBase) ZoneList() []string {
+	zoneStr := input.Zone
+	if len(zoneStr) > 0 {
+		input.Zones = append(input.Zones, zoneStr)
+	}
+	return input.Zones
+}
+
+func (input ZonalFilterListBase) FirstZone() string {
+	if len(input.Zone) > 0 {
+		return input.Zone
+	}
+	if len(input.Zones) > 0 {
+		return input.Zones[0]
+	}
+	return ""
+}
 func (input ZonalFilterListInput) ZoneList() []string {
 	zoneStr := input.Zone
 	if len(zoneStr) > 0 {

pkg/apis/compute/guestnetwork.go

@@ -21,6 +21,8 @@ type GuestnetworkDetails struct {
 
 	// IP子网名称
 	Network string `json:"network"`
+	// 所属Wire
+	WireId string `json:"wire_id"`
 }
 
 type GuestnetworkShortDesc struct {

pkg/apis/compute/guests.go

@@ -69,8 +69,8 @@ type ServerListInput struct {
 	// 按主机资源类型进行排序
 	// enum: shared,prepaid,dedicated
 	ResourceType string `json:"resource_type"`
-	// 返回开启主备机功能的主机
-	GetBackupGuestsOnHost *bool `json:"get_backup_guests_on_host"`
+	// 返回该宿主机上的所有虚拟机，包括备份机
+	GetAllGuestsOnHost string `json:"get_all_guests_on_host"`
 
 	// 根据宿主机 SN 过滤
 	// HostSn string `json:"host_sn"`
@@ -340,3 +340,8 @@ type GuestSaveToTemplateInput struct {
 	// The generate name of guest template
 	GenerateName string `json:"generate_name"`
 }
+
+type GuestSyncFixNicsInput struct {
+	// 需要修正的IP地址列表
+	Ip []string `json:"ip"`
+}

pkg/apis/compute/guesttemplate.go

@@ -57,6 +57,7 @@ type GuestTemplateDetails struct {
 
 	Secgroups []string `json:"secgroups"`
 	Zone      string   `json:"zone"`
+	ZoneId    string   `json:"zone_id"`
 	Brand     string   `json:"brand"`
 
 	ConfigInfo GuestTemplateConfigInfo `json:"config_info"`
@@ -71,9 +72,7 @@ type GuestTemplateListInput struct {
 }
 
 type GuestTemplateConfigInfo struct {
-	Region               string                 `json:"region"`
-	Zone                 string                 `json:"zone"`
-	Hypervisor           string                 `json:"hypervisor"`
+	Metadata             map[string]string      `json:"metadata"`
 	Secgroup             string                 `json:"secgroup"`
 	Sku                  GuestTemplateSku       `json:"sku"`
 	Disks                []GuestTemplateDisk    `json:"disks"`

pkg/apis/compute/host.go

@@ -108,6 +108,8 @@ type HostListInput struct {
 	Uuid []string `json:"uuid"`
 	// 主机启动模式, 可能值位PXE和ISO
 	BootMode []string `json:"boot_mode"`
+	// 虚拟机所在的二层网络
+	ServerIdForNetwork string `json:"server_id_for_network"`
 }
 
 type HostDetails struct {
@@ -119,9 +121,10 @@ type HostDetails struct {
 
 	Schedtags []SchedtagShortDescDetails `json:"schedtags"`
 
-	ServerId  string `json:"server_id"`
-	Server    string `json:"server"`
-	ServerIps string `json:"server_ips"`
+	ServerId             string `json:"server_id"`
+	Server               string `json:"server"`
+	ServerIps            string `json:"server_ips"`
+	ServerPendingDeleted bool   `json:"server_pending_deleted"`
 	// 网卡数量
 	NicCount int `json:"nic_count"`
 	// 网卡详情
@@ -138,7 +141,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率
@@ -164,7 +167,8 @@ type HostDetails struct {
 	CanPrepare        bool                `json:"can_prepare"`
 	PrepareFailReason string              `json:"prepare_fail_reason"`
 	// 允许开启宿主机健康检查
-	AllowHealthCheck bool `json:"allow_health_check"`
+	AllowHealthCheck      bool `json:"allow_health_check"`
+	AutoMigrateOnHostDown bool `json:"auto_migrate_on_host_down"`
 
 	// reserved resource for isolated device
 	ReservedResourceForGpu IsolatedDeviceReservedResourceInput `json:"reserved_resource_for_gpu"`

pkg/apis/compute/input.go

@@ -61,8 +61,7 @@ type CachedimageListInput struct {
 }
 
 type ExternalProjectListInput struct {
-	apis.StandaloneResourceListInput
-	apis.ProjectizedResourceListInput
+	apis.VirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput

pkg/apis/compute/instance_snapshot.go

@@ -50,6 +50,9 @@ type InstanceSnapshotDetails struct {
 	// 快照列表
 	Snapshots  []SimpleSnapshot  `json:"snapshots"`
 	Properties map[string]string `json:"properties"`
+
+	// 主机快照大小
+	Size int `json:"size"`
 }
 
 type InstanceSnapshotListInput struct {

pkg/apis/compute/loadbalancer.go

@@ -170,6 +170,8 @@ type LoadbalancerDetails struct {
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
+	LoadbalancerClusterResourceInfo
+
 	VpcResourceInfoBase
 	ZoneResourceInfoBase
 	NetworkResourceInfoBase

pkg/apis/compute/loadbalanceracl.go

@@ -23,3 +23,24 @@ type LoadbalancerAclDetails struct {
 
 	SLoadbalancerAcl
 }
+
+type LoadbalancerAclResourceInfo struct {
+	// 负载均衡ACL名称
+	Acl string `json:"acl"`
+}
+
+type LoadbalancerAclResourceInput struct {
+	// ACL名称或ID
+	AclId string `json:"acl_id"`
+
+	// swagger:ignore
+	// Deprecated
+	Acl string `json:"acl" "yunion:deprecated-by":"acl_id"`
+}
+
+type LoadbalancerAclFilterListInput struct {
+	LoadbalancerAclResourceInput
+
+	// 以ACL名称排序
+	OrderByAcl string `json:"order_by_acl"`
+}

pkg/apis/compute/loadbalancercertificate.go

@@ -20,3 +20,24 @@ type LoadbalancerCertificateDetails struct {
 	apis.VirtualResourceDetails
 	SLoadbalancerCertificate
 }
+
+type LoadbalancerCertificateResourceInfo struct {
+	// 负载均衡证书名称
+	Certificate string `json:"certificate"`
+}
+
+type LoadbalancerCertificateResourceInput struct {
+	// 证书名称或ID
+	Certificate string `json:"certificate"`
+
+	// swagger:ignore
+	// Deprecated
+	CertificateId string `json:"certificate_id" "yunion:deprecated-by":"certificate"`
+}
+
+type LoadbalancerCertificateFilterListInput struct {
+	LoadbalancerCertificateResourceInput
+
+	// 以证书名称排序
+	OrderByCertificate string `json:"order_by_certificate"`
+}

pkg/apis/compute/loadbalancerlistener.go

@@ -19,11 +19,12 @@ import "yunion.io/x/onecloud/pkg/apis"
 type LoadbalancerListenerDetails struct {
 	apis.VirtualResourceDetails
 	LoadbalancerResourceInfo
+	LoadbalancerAclResourceInfo
+	LoadbalancerCertificateResourceInfo
 
 	SLoadbalancerListener
 
 	BackendGroup        string `json:"backend_group"`
-	AclName             string `json:"acl_name"`
 	CertificateName     string `json:"certificate_name"`
 	OriginCertificateId string `json:"origin_certificate_id"`
 }

pkg/apis/compute/scaling_group.go

@@ -134,9 +134,15 @@ type ScalingGroupDetails struct {
 	// example: OneCloud
 	Brand string `json:"brand"`
 
-	// description: 网络ID
-	// example: net-12345
-	Networks []string `json:"networks"`
+	// description: 网络信息
+	Networks []ScalingGroupNetwork `json:"networks"`
+}
+
+type ScalingGroupNetwork struct {
+	Id           string `json:"id"`
+	Name         string `json:"name"`
+	GuestIpStart string `json:"guest_ip_start"`
+	GuestIpEnd   string `json:"guest_ip_end"`
 }
 
 type ScalingGroupResourceInfo struct {

pkg/apis/compute/schedtag.go

@@ -65,7 +65,7 @@ type SchedtagListInput struct {
 	apis.ScopedResourceBaseListInput
 
 	// fitler by resource_type
-	ResourceType string `json:"resource_type"`
+	ResourceType []string `json:"resource_type"`
 	// swagger:ignore
 	// Deprecated
 	// filter by type, alias for resource_type
@@ -86,6 +86,7 @@ type SchedtagDetails struct {
 	HostCount        int    `json:"host_count"`
 	ServerCount      int    `json:"server_count"`
 	OtherCount       int    `json:"other_count"`
+	ResourceCount    int    `json:"resource_count"`
 	JoinModelKeyword string `json:"join_model_keyword"`
 }
 

pkg/apis/compute/secgroup.go

@@ -145,8 +145,18 @@ type SecgroupListInput struct {
 	// pattern:asc|desc
 	OrderByGuestCnt string `json:"order_by_guest_cnt"`
 
-	// 是否被修改
-	IsDirty *bool `json:"is_dirty"`
+	// 模糊过滤规则中含有指定ip的安全组
+	// example: 10.10.2.1
+	Ip string `json:"ip"`
+
+	// 精确匹配规则中含有指定端口的安全组
+	// example: 100-200
+	Ports string `json:"ports"`
+
+	// 指定过滤规则的方向(仅在指定ip或ports时生效) choices: all|in|out
+	// default: all
+	// example: in
+	Direction string `json:"direction"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -164,12 +174,18 @@ type SecurityGroupRuleListInput struct {
 	apis.ResourceBaseListInput
 	SecgroupFilterListInput
 
+	Projects []string `json:"projects"`
+
 	// 以direction字段过滤安全组规则
 	Direction string `json:"direction"`
 	// 以action字段过滤安全组规则
 	Action string `json:"action"`
 	// 以protocol字段过滤安全组规则
 	Protocol string `json:"protocol"`
+	// 以ports字段过滤安全组规则
+	Ports string `json:"ports"`
+	// 根据ip模糊匹配安全组规则
+	Ip string `json:"ip"`
 }
 
 type SecgroupResourceInput struct {
@@ -179,6 +195,9 @@ type SecgroupResourceInput struct {
 	// Deprecated
 	// filter by secgroup_id
 	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
+
+	// 模糊匹配安全组规则名称
+	SecgroupName string `json:"secgroup_name"`
 }
 
 type SecgroupFilterListInput struct {
@@ -193,15 +212,22 @@ type SecgroupDetails struct {
 	SSecurityGroup
 
 	// 关联云主机数量
-	GuestCnt int `json:"guest_cnt"`
+	GuestCnt int `json:"guest_cnt,allowempty"`
+
+	// 关联此安全组的云主机is_system为true数量
+	SystemGuestCnt int `json:"system_guest_cnt,allowempty"`
+
+	// admin_secgrp_id为此安全组的云主机数量
+	AdminGuestCnt int `json:"admin_guest_cnt,allowempty"`
+
 	// 安全组缓存数量
-	CacheCnt int `json:"cache_cnt"`
+	CacheCnt int `json:"cache_cnt,allowempty"`
 	// 规则信息
-	Rules string `json:"rules"`
+	Rules []SSecurityGroupRule `json:"rules"`
 	// 入方向规则信息
-	InRules string `json:"in_rules"`
+	InRules []SSecurityGroupRule `json:"in_rules"`
 	// 出方向规则信息
-	OutRules string `json:"out_rules"`
+	OutRules []SSecurityGroupRule `json:"out_rules"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/storage_const.go

@@ -127,7 +127,7 @@ var (
 		STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS,
 	}
 
-	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA}
 
 	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
 

pkg/apis/compute/zz_generated.model.go

@@ -202,8 +202,6 @@ type SCloudaccount struct {
 	ProxySettingId string `json:"proxy_setting_id"`
 	// 公有云子账号登录地址
 	IamLoginUrl string `json:"iam_login_url"`
-	// 是否支持创建公有云子账号
-	IsSupportCloudId *bool `json:"is_support_cloud_id,omitempty"`
 }
 
 // SCloudprovider is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SCloudprovider.
@@ -731,10 +729,11 @@ type SElasticip struct {
 
 // SExternalProject is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SExternalProject.
 type SExternalProject struct {
-	apis.SStandaloneResourceBase
-	apis.SProjectizedResourceBase
+	apis.SVirtualResourceBase
 	apis.SExternalizedResourceBase
 	SManagedResourceBase
+	// 归属云账号ID
+	CloudaccountId string `json:"cloudaccount_id"`
 }
 
 // SGlobalVpc is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGlobalVpc.
@@ -868,6 +867,20 @@ type SGuestTemplateResourceBase struct {
 	GuestTemplateId string `json:"guest_template_id"`
 }
 
+// SGuestcdrom is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestcdrom.
+type SGuestcdrom struct {
+	Id string `json:"id"`
+	// = Column(VARCHAR(36, charset='ascii'), primary_key=True)
+	ImageId string `json:"image_id"`
+	// Column(VARCHAR(36, charset='ascii'), nullable=True)
+	Name string `json:"name"`
+	// Column(VARCHAR(64, charset='ascii'), nullable=True)
+	Path string `json:"path"`
+	// Column(VARCHAR(256, charset='ascii'), nullable=True)
+	Size          int `json:"size"`
+	UpdateVersion int `json:"update_version"`
+}
+
 // SGuestdisk is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestdisk.
 type SGuestdisk struct {
 	SGuestJointsBase
@@ -987,11 +1000,10 @@ type SHost struct {
 	OvnVersion  string `json:"ovn_version"`
 	IsBaremetal bool   `json:"is_baremetal"`
 	// 是否处于维护状态
-	IsMaintenance     bool      `json:"is_maintenance"`
-	LastPingAt        time.Time `json:"last_ping_at"`
-	EnableHealthCheck bool      `json:"enable_health_check"`
-	ResourceType      string    `json:"resource_type"`
-	RealExternalId    string    `json:"real_external_id"`
+	IsMaintenance     bool   `json:"is_maintenance"`
+	EnableHealthCheck bool   `json:"enable_health_check"`
+	ResourceType      string `json:"resource_type"`
+	RealExternalId    string `json:"real_external_id"`
 	// 是否为导入的宿主机
 	IsImport bool `json:"is_import"`
 	// 是否允许PXE启动
@@ -1179,8 +1191,7 @@ type SLoadbalancer struct {
 	AddressType string `json:"address_type"`
 	// 网络类型
 	NetworkType string `json:"network_type"`
-	// 负载均衡集群Id
-	ClusterId string `json:"cluster_id"`
+	SLoadbalancerClusterResourceBase
 	// 计费类型
 	ChargeType string `json:"charge_type"`
 	// 套餐名称
@@ -1246,12 +1257,9 @@ type SLoadbalancerAgentDeployment struct {
 
 // SLoadbalancerAgentParams is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SLoadbalancerAgentParams.
 type SLoadbalancerAgentParams struct {
-	KeepalivedConfTmpl string                           `json:"keepalived_conf_tmpl"`
-	HaproxyConfTmpl    string                           `json:"haproxy_conf_tmpl"`
-	TelegrafConfTmpl   string                           `json:"telegraf_conf_tmpl"`
-	Vrrp               SLoadbalancerAgentParamsVrrp     `json:"vrrp"`
-	Haproxy            SLoadbalancerAgentParamsHaproxy  `json:"haproxy"`
-	Telegraf           SLoadbalancerAgentParamsTelegraf `json:"telegraf"`
+	KeepalivedConfTmpl string `json:"keepalived_conf_tmpl"`
+	HaproxyConfTmpl    string `json:"haproxy_conf_tmpl"`
+	TelegrafConfTmpl   string `json:"telegraf_conf_tmpl"`
 }
 
 // SLoadbalancerAgentParamsHaproxy is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SLoadbalancerAgentParamsHaproxy.
@@ -1377,7 +1385,7 @@ type SLoadbalancerHTTPRedirect struct {
 
 // SLoadbalancerHTTPSListener is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SLoadbalancerHTTPSListener.
 type SLoadbalancerHTTPSListener struct {
-	CertificateId       string `json:"certificate_id"`
+	SLoadbalancerCertificateResourceBase
 	CachedCertificateId string `json:"cached_certificate_id"`
 	TLSCipherPolicy     string `json:"tls_cipher_policy"`
 	EnableHttp2         bool   `json:"enable_http2"`
@@ -1425,8 +1433,8 @@ type SLoadbalancerListener struct {
 	BackendIdleTimeout int    `json:"backend_idle_timeout"`
 	AclStatus          string `json:"acl_status"`
 	AclType            string `json:"acl_type"`
-	AclId              string `json:"acl_id"`
-	CachedAclId        string `json:"cached_acl_id"`
+	SLoadbalancerAclResourceBase
+	CachedAclId string `json:"cached_acl_id"`
 	SLoadbalancerRateLimiter
 	SLoadbalancerTCPListener
 	SLoadbalancerUDPListener
@@ -1493,6 +1501,16 @@ type SManagedResourceBase struct {
 	ManagerId string `json:"manager_id"`
 }
 
+// SNatDEntry is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SNatDEntry.
+type SNatDEntry struct {
+	SNatEntry
+	ExternalIP   string `json:"external_ip"`
+	ExternalPort int    `json:"external_port"`
+	InternalIP   string `json:"internal_ip"`
+	InternalPort int    `json:"internal_port"`
+	IpProtocol   string `json:"ip_protocol"`
+}
+
 // SNatEntry is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SNatEntry.
 type SNatEntry struct {
 	apis.SStatusInfrasResourceBase
@@ -1523,6 +1541,25 @@ type SNatgatewayResourceBase struct {
 	NatgatewayId string `json:"natgateway_id"`
 }
 
+// SNetInterface is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SNetInterface.
+type SNetInterface struct {
+	Mac string `json:"mac"`
+	// Column(VARCHAR(36, charset='ascii'), primary_key=True)
+	BaremetalId string `json:"baremetal_id"`
+	// Column(VARCHAR(36, charset='ascii'), nullable=True)
+	WireId string `json:"wire_id"`
+	// Column(VARCHAR(36, charset='ascii'), nullable=True)
+	Rate int `json:"rate"`
+	// Column(Integer, nullable=True) # Mbps
+	NicType string `json:"nic_type"`
+	// Column(VARCHAR(36, charset='ascii'), nullable=True)
+	Index byte `json:"index"`
+	// Column(TINYINT, nullable=True)
+	LinkUp bool `json:"link_up"`
+	// Column(Boolean, nullable=True)
+	Mtu int16 `json:"mtu"`
+}
+
 // SNetwork is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SNetwork.
 type SNetwork struct {
 	apis.SSharableVirtualResourceBase
@@ -1697,8 +1734,6 @@ type SScalingAlarm struct {
 	Value    float64 `json:"value"`
 	// Real-time cumulate number
 	RealCumulate int `json:"real_cumulate"`
-	// Last trigger time
-	LastTriggerTime time.Time `json:"last_trigger_time"`
 }
 
 // SScalingGroup is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SScalingGroup.
@@ -1729,10 +1764,6 @@ type SScalingGroup struct {
 	HealthCheckGov            int    `json:"health_check_gov"`
 	LoadbalancerBackendPort   int    `json:"loadbalancer_backend_port"`
 	LoadbalancerBackendWeight int    `json:"loadbalancer_backend_weight"`
-	// Time to allow scale
-	AllowScaleTime time.Time `json:"allow_scale_time"`
-	// NextCheckTime descripe the next time to check instance's health
-	NextCheckTime time.Time `json:"next_check_time"`
 }
 
 // SScalingGroupGuest is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SScalingGroupGuest.
@@ -1856,7 +1887,8 @@ type SSecurityGroupCache struct {
 	SManagedResourceBase
 	SSecurityGroupResourceBase
 	// 虚拟私有网络外部Id
-	VpcId string `json:"vpc_id"`
+	VpcId             string `json:"vpc_id"`
+	ExternalProjectId string `json:"external_project_id"`
 }
 
 // SSecurityGroupResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SSecurityGroupResourceBase.

pkg/apis/identity/input.go

@@ -151,6 +151,9 @@ type ProjectListInput struct {
 
 	UserFilterListInput
 	GroupFilterListInput
+
+	// 过滤出指定用户或者组可以加入的项目
+	Jointable *bool `json:"jointable"`
 }
 
 type DomainListInput struct {

pkg/apis/identity/zz_generated.model.go

@@ -92,6 +92,20 @@ type SFederatedUser struct {
 	DisplayName string `json:"display_name"`
 }
 
+// SFederationProtocol is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SFederationProtocol.
+type SFederationProtocol struct {
+	Id        string `json:"id"`
+	IdpId     string `json:"idp_id"`
+	MappingId string `json:"mapping_id"`
+}
+
+// SFernetKey is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SFernetKey.
+type SFernetKey struct {
+	Type  string `json:"type"`
+	Index int    `json:"index"`
+	Key   string `json:"key"`
+}
+
 // SGroup is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SGroup.
 type SGroup struct {
 	SIdentityBaseResource
@@ -129,28 +143,45 @@ type SIdmapping struct {
 	EntityType  string `json:"entity_type"`
 }
 
+// SIdpRemoteIds is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SIdpRemoteIds.
+type SIdpRemoteIds struct {
+	IdpId    string `json:"idp_id"`
+	RemoteId string `json:"remote_id"`
+}
+
+// SImpliedRole is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SImpliedRole.
+type SImpliedRole struct {
+	PriorRoleId   string `json:"prior_role_id"`
+	ImpliedRoleId string `json:"implied_role_id"`
+}
+
 // SLocalUser is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SLocalUser.
 type SLocalUser struct {
 	apis.SResourceBase
-	Id              int       `json:"id"`
-	UserId          string    `json:"user_id"`
-	DomainId        string    `json:"domain_id"`
-	Name            string    `json:"name"`
-	FailedAuthCount int       `json:"failed_auth_count"`
-	FailedAuthAt    time.Time `json:"failed_auth_at"`
+	Id              int    `json:"id"`
+	UserId          string `json:"user_id"`
+	DomainId        string `json:"domain_id"`
+	Name            string `json:"name"`
+	FailedAuthCount int    `json:"failed_auth_count"`
+}
+
+// SNonlocalUser is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SNonlocalUser.
+type SNonlocalUser struct {
+	DomainId string `json:"domain_id"`
+	Name     string `json:"name"`
+	UserId   string `json:"user_id"`
 }
 
 // SPassword is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SPassword.
 type SPassword struct {
 	apis.SResourceBase
-	Id           int       `json:"id"`
-	LocalUserId  int       `json:"local_user_id"`
-	Password     string    `json:"password"`
-	ExpiresAt    time.Time `json:"expires_at"`
-	SelfService  bool      `json:"self_service"`
-	PasswordHash string    `json:"password_hash"`
-	CreatedAtInt int64     `json:"created_at_int"`
-	ExpiresAtInt int64     `json:"expires_at_int"`
+	Id           int    `json:"id"`
+	LocalUserId  int    `json:"local_user_id"`
+	Password     string `json:"password"`
+	SelfService  bool   `json:"self_service"`
+	PasswordHash string `json:"password_hash"`
+	CreatedAtInt int64  `json:"created_at_int"`
+	ExpiresAtInt int64  `json:"expires_at_int"`
 }
 
 // SPolicy is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SPolicy.
@@ -168,6 +199,12 @@ type SProject struct {
 	IsDomain *bool  `json:"is_domain,omitempty"`
 }
 
+// SProjectExtended is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SProjectExtended.
+type SProjectExtended struct {
+	SProject
+	DomainName string `json:"domain_name"`
+}
+
 // SRegion is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SRegion.
 type SRegion struct {
 	apis.SStandaloneResourceBase
@@ -181,6 +218,17 @@ type SRole struct {
 	apis.SSharableBaseResource
 }
 
+// SScopeResource is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SScopeResource.
+type SScopeResource struct {
+	DomainId  string `json:"domain_id"`
+	ProjectId string `json:"project_id"`
+	OwnerId   string `json:"owner_id"`
+	RegionId  string `json:"region_id"`
+	ServiceId string `json:"service_id"`
+	Resource  string `json:"resource"`
+	Count     int    `json:"count"`
+}
+
 // SService is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SService.
 type SService struct {
 	apis.SStandaloneResourceBase
@@ -213,6 +261,13 @@ type SUser struct {
 	EnableMfa        *bool  `json:"enable_mfa,omitempty"`
 }
 
+// SUserOption is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SUserOption.
+type SUserOption struct {
+	UserId      string `json:"user_id"`
+	OptionId    string `json:"option_id"`
+	OptionValue string `json:"option_value"`
+}
+
 // SUsergroupMembership is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SUsergroupMembership.
 type SUsergroupMembership struct {
 	apis.SResourceBase

pkg/apis/image/consts.go

@@ -50,6 +50,7 @@ const (
 	IMAGE_IS_READONLY         = "is_readonly"
 	IMAGE_PARTITION_TYPE      = "partition_type"
 	IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init"
+	IMAGE_DISABLE_USB_KBD     = "disable_usb_kbd"
 
 	IMAGE_STATUS_UPDATING = "updating"
 )

pkg/apis/image/zz_generated.model.go

@@ -62,9 +62,44 @@ type SImage struct {
 	OssChecksum string `json:"oss_checksum"`
 }
 
+// SImageMember is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImageMember.
+type SImageMember struct {
+	SImagePeripheral
+	Member   string `json:"member"`
+	CanShare bool   `json:"can_share"`
+}
+
 // SImagePeripheral is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImagePeripheral.
 type SImagePeripheral struct {
 	apis.SResourceBase
 	Id      int    `json:"id"`
 	ImageId string `json:"image_id"`
 }
+
+// SImageProperty is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImageProperty.
+type SImageProperty struct {
+	SImagePeripheral
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// SImageSubformat is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImageSubformat.
+type SImageSubformat struct {
+	SImagePeripheral
+	Format          string `json:"format"`
+	Size            int64  `json:"size"`
+	Location        string `json:"location"`
+	Checksum        string `json:"checksum"`
+	FastHash        string `json:"fast_hash"`
+	Status          string `json:"status"`
+	TorrentSize     int64  `json:"torrent_size"`
+	TorrentLocation string `json:"torrent_location"`
+	TorrentChecksum string `json:"torrent_checksum"`
+	TorrentStatus   string `json:"torrent_status"`
+}
+
+// SImageTag is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImageTag.
+type SImageTag struct {
+	SImagePeripheral
+	Value string `json:"value"`
+}

pkg/apis/input.go

@@ -75,9 +75,11 @@ type SharableVirtualResourceCreateInput struct {
 	SharableResourceBaseCreateInput
 }
 
-type StatusUserResourceCreateInput struct {
-	StatusBaseResourceCreateInput
-	UserResourceCreateInput
+type StatusDomainLevelUserResourceCreateInput struct {
+	StatusDomainLevelResourceCreateInput
+
+	// 本地用户Id，若为空则使用当前用户Id作为此参数值
+	OwnerId string `json:"owner_id"`
 }
 
 type UserResourceCreateInput struct {

pkg/apis/list.go

@@ -53,16 +53,24 @@ type ProjectizedResourceListInput struct {
 	OrderByTenant string `json:"order_by_tenant" "yunion:deprecated-by":"order_by_project"`
 }
 
-type StatusUserResourceListInput struct {
-	StatusResourceBaseListInput
-	UserResourceListInput
+type StatusDomainLevelUserResourceListInput struct {
+	StatusDomainLevelResourceListInput
+
+	// 查询指定的用户（ID或名称）拥有的资源
+	User string `json:"user"`
+	// swagger:ignore
+	// Deprecated
+	// Filter by userId
+	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
 }
 
 type UserResourceListInput struct {
 	StandaloneResourceListInput
 	ScopedResourceInput
 
-	// list in admin mode
+	// swagger:ignore
+	// Is an admin call? equivalent to scope=system
+	// Deprecated
 	Admin *bool `json:"admin"`
 
 	// 查询指定的用户（ID或名称）拥有的资源

pkg/apis/monitor/commalert.go

@@ -23,6 +23,8 @@ const (
 	CommonAlertFieldOpt_Division = "/"
 )
 
+var CommonAlertLevels = []string{"normal", "important", "fatal"}
+
 type CommonAlertCreateInput struct {
 	CommonMetricInputQuery
 	AlertCreateInput

pkg/apis/monitor/zz_generated.model.go

@@ -79,6 +79,16 @@ type SDataSource struct {
 	IsDefault *bool  `json:"is_default,omitempty"`
 }
 
+// SMeterAlert is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SMeterAlert.
+type SMeterAlert struct {
+	SV1Alert
+}
+
+// SNodeAlert is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SNodeAlert.
+type SNodeAlert struct {
+	SV1Alert
+}
+
 // SNotification is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SNotification.
 type SNotification struct {
 	apis.SVirtualResourceBase
@@ -124,3 +134,8 @@ type SSuggestSysRule struct {
 	Setting  interface{} `json:"setting"`
 	ExecTime time.Time   `json:"exec_time"`
 }
+
+// SV1Alert is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SV1Alert.
+type SV1Alert struct {
+	SAlert
+}

pkg/apis/output.go

@@ -121,8 +121,11 @@ type MetadataResourceInfo struct {
 	Metadata map[string]string `json:"metadata"`
 }
 
-type StatusUserResourceDetails struct {
-	UserResourceDetails
+type StatusDomainLevelUserResourceDetails struct {
+	StatusDomainLevelResourceDetails
+
+	// 用户名称
+	OwnerName string `json:"owner_name"`
 }
 
 type UserResourceDetails struct {

pkg/apis/zz_generated.model.go

@@ -99,9 +99,60 @@ type SJointResourceBase struct {
 // SKeystoneCacheObject is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SKeystoneCacheObject.
 type SKeystoneCacheObject struct {
 	SStandaloneResourceBase
-	DomainId  string    `json:"domain_id"`
-	Domain    string    `json:"domain"`
-	LastCheck time.Time `json:"last_check"`
+	DomainId string `json:"domain_id"`
+	Domain   string `json:"domain"`
+}
+
+// SMetadata is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SMetadata.
+type SMetadata struct {
+	// 资源类型
+	// example: network
+	ObjType string `json:"obj_type"`
+	// 资源ID
+	// example: 87321a70-1ecb-422a-8b0c-c9aa632a46a7
+	ObjId string `json:"obj_id"`
+	// 资源组合ID
+	// example: network::87321a70-1ecb-422a-8b0c-c9aa632a46a7
+	Id string `json:"id"`
+	// 标签KEY
+	// exmaple: 部门
+	Key string `json:"key"`
+	// 标签值
+	// example: 技术部
+	Value string `json:"value"`
+	// 是否被删除
+	Deleted bool `json:"deleted"`
+}
+
+// SOpsLog is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SOpsLog.
+type SOpsLog struct {
+	Id int64 `json:"id"`
+	// = Column(BigInteger, primary_key=True)
+	ObjType string `json:"obj_type"`
+	// = Column(VARCHAR(40, charset='ascii'), nullable=False)
+	ObjId string `json:"obj_id"`
+	//  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
+	ObjName string `json:"obj_name"`
+	// = Column(VARCHAR(128, charset='utf8'), nullable=False)
+	Action string `json:"action"`
+	// = Column(VARCHAR(32, charset='ascii'), nullable=False)
+	Notes     string `json:"notes"`
+	ProjectId string `json:"tenant_id"`
+	// = Column(VARCHAR(ID_LENGTH, charset='ascii'))
+	Project         string `json:"tenant"`
+	ProjectDomainId string `json:"project_domain_id"`
+	ProjectDomain   string `json:"project_domain"`
+	UserId          string `json:"user_id"`
+	// = Column(VARCHAR(ID_LENGTH, charset='ascii'))
+	User string `json:"user"`
+	// = Column(VARCHAR(128, charset='utf8'))
+	DomainId string `json:"domain_id"`
+	Domain   string `json:"domain"`
+	Roles    string `json:"roles"`
+	// BillingType    string    `width:"64" charset:"ascii" default:"postpaid" list:"user" create:"user"`      // billing_type = Column(VARCHAR(64, charset='ascii'), nullable=True)
+	OpsTime        time.Time `json:"ops_time"`
+	OwnerDomainId  string    `json:"owner_domain_id"`
+	OwnerProjectId string    `json:"owner_tenant_id"`
 }
 
 // SProjectizedResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SProjectizedResourceBase.
@@ -178,6 +229,13 @@ type SStatusDomainLevelResourceBase struct {
 	SStatusResourceBase
 }
 
+// SStatusDomainLevelUserResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SStatusDomainLevelUserResourceBase.
+type SStatusDomainLevelUserResourceBase struct {
+	SStatusDomainLevelResourceBase
+	// 本地用户Id
+	OwnerId string `json:"owner_id"`
+}
+
 // SStatusInfrasResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SStatusInfrasResourceBase.
 type SStatusInfrasResourceBase struct {
 	SInfrasResourceBase
@@ -196,10 +254,14 @@ type SStatusStandaloneResourceBase struct {
 	SStatusResourceBase
 }
 
-// SStatusUserResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SStatusUserResourceBase.
-type SStatusUserResourceBase struct {
-	SUserResourceBase
-	SStatusResourceBase
+// STenant is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.STenant.
+type STenant struct {
+	SKeystoneCacheObject
+}
+
+// SUser is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SUser.
+type SUser struct {
+	SKeystoneCacheObject
 }
 
 // SUserResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SUserResourceBase.

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/app/auth.go

@@ -84,8 +84,9 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
-		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
-			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,
+		policy.EnableGlobalRbac(
+			time.Second*time.Duration(options.RbacPolicySyncPeriodSeconds),
+			time.Second*time.Duration(options.RbacPolicySyncFailedRetrySeconds),
 			options.RbacDebug,
 		)
 	}

pkg/cloudcommon/database.go

@@ -27,8 +27,6 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
-	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
-	"yunion.io/x/onecloud/pkg/cloudcommon/informer"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 )
 
@@ -80,6 +78,7 @@ func InitDB(options *common_options.DBOptions) {
 	}
 	// lm := lockman.NewNoopLockManager()
 
+	/* disable informer feature on release/3.3
 	if len(options.EtcdEndpoints) != 0 {
 		log.Infof("using etcd as resource informer backend")
 		tlsCfg, err := options.GetEtcdTLSConfig()
@@ -99,6 +98,7 @@ func InitDB(options *common_options.DBOptions) {
 		}
 		informer.Init(informerBackend)
 	}
+	*/
 }
 
 func CloseDB() {

pkg/cloudcommon/db/db_dispatcher.go

@@ -857,6 +857,11 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 	modelValue := reflect.ValueOf(dispatcher.modelManager)
 	params := []interface{}{ctx, userCred, query}
 
+	funcValue := modelValue.MethodByName(funcName)
+	if !funcValue.IsValid() || funcValue.IsNil() {
+		return nil, nil
+	}
+
 	if consts.IsRbacEnabled() {
 		ownerId, err := fetchOwnerId(ctx, dispatcher.modelManager, userCred, query)
 		if err != nil {
@@ -883,10 +888,6 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 		}
 	}
 
-	funcValue := modelValue.MethodByName(funcName)
-	if !funcValue.IsValid() || funcValue.IsNil() {
-		return nil, nil
-	}
 	outs, err := callFunc(funcValue, funcName, params...)
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("reflect call %s fail %s", funcName, err)
@@ -1483,6 +1484,9 @@ func (dispatcher *DBModelDispatcher) PerformAction(ctx context.Context, idStr st
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 
+	if err := model.PreCheckPerformAction(ctx, userCred, action, query, data); err != nil {
+		return nil, err
+	}
 	return objectPerformAction(dispatcher, model, reflect.ValueOf(model), ctx, userCred, action, query, data)
 }
 

pkg/cloudcommon/db/domain.go

@@ -77,6 +77,10 @@ func (model *SDomainizedResourceBase) GetChangeOwnerCandidateDomainIds() []strin
 	return nil
 }
 
+func (model *SDomainizedResourceBase) GetChangeOwnerRequiredDomainIds() []string {
+	return nil
+}
+
 func ValidateCreateDomainId(domainId string) error {
 	if !consts.GetNonDefaultDomainProjects() && domainId != identity.DEFAULT_DOMAIN_ID {
 		return httperrors.NewForbiddenError("project in non-default domain is prohibited")

pkg/cloudcommon/db/domainresource.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -133,6 +134,9 @@ func (model *SDomainLevelResourceBase) AllowPerformChangeOwner(ctx context.Conte
 }
 
 func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformChangeDomainOwnerInput) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.GetIStandaloneModel().IsShared() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot change owner of shared resource")
 	}
@@ -162,6 +166,11 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 	if len(candidates) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), candidates) {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner candidate list")
 	}
+	requires := model.GetIDomainLevelModel().GetChangeOwnerRequiredDomainIds()
+	log.Debugf("%s required domains: %s", model.Keyword(), requires)
+	if len(requires) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), requires) {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner required list")
+	}
 
 	if !IsAdminAllowPerform(userCred, model, "change-owner") {
 		return nil, errors.Wrap(httperrors.ErrNotSufficientPrivilege, "require system privileges")

pkg/cloudcommon/db/external.go

@@ -27,7 +27,7 @@ import (
 // +onecloud:model-api-gen
 type SExternalizedResourceBase struct {
 	// 外部Id, 对用公有云私有资源自身的Id
-	ExternalId string `width:"256" charset:"utf8" index:"true" list:"user" create:"admin_optional" update:"admin" json:"external_id"`
+	ExternalId string `width:"256" charset:"utf8" index:"true" list:"user" create:"domain_optional" update:"admin" json:"external_id"`
 }
 
 type SExternalizedResourceBaseManager struct{}
@@ -67,7 +67,14 @@ func SetExternalId(model IExternalizedModel, userCred mcclient.TokenCredential,
 }
 
 func FetchByExternalId(manager IModelManager, idStr string) (IExternalizedModel, error) {
+	return FetchByExternalIdAndManagerId(manager, idStr, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+		return q
+	})
+}
+
+func FetchByExternalIdAndManagerId(manager IModelManager, idStr string, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (IExternalizedModel, error) {
 	q := manager.Query().Equals("external_id", idStr)
+	q = filter(q)
 	count, err := q.CountWithError()
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/infraresource.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -200,6 +201,9 @@ func (model *SInfrasResourceBase) PerformChangeOwner(
 	query jsonutils.JSONObject,
 	input apis.PerformChangeDomainOwnerInput,
 ) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.IsShared() {
 		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot change owner when shared!")
 	}
@@ -248,6 +252,15 @@ func (model *SInfrasResourceBase) SaveSharedInfo(src apis.TOwnerSource, ctx cont
 }
 
 func (model *SInfrasResourceBase) SyncShareState(ctx context.Context, userCred mcclient.TokenCredential, shareInfo apis.SAccountShareInfo) {
+	if !consts.GetNonDefaultDomainProjects() {
+		if model.PublicSrc != string(apis.OWNER_SOURCE_LOCAL) {
+			model.SaveSharedInfo(apis.OWNER_SOURCE_CLOUD, ctx, userCred, apis.SShareInfo{
+				IsPublic:    true,
+				PublicScope: rbacutils.ScopeSystem,
+			})
+		}
+		return
+	}
 	si := shareInfo.GetDomainShareInfo()
 	if model.PublicSrc != string(apis.OWNER_SOURCE_LOCAL) {
 		model.SaveSharedInfo(apis.OWNER_SOURCE_CLOUD, ctx, userCred, si)

pkg/cloudcommon/db/interface.go

@@ -162,6 +162,7 @@ type IModel interface {
 	// allow perform action
 	AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
 	PerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+	PreCheckPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 
 	// update hooks
 	ValidateUpdateCondition(ctx context.Context) error

pkg/cloudcommon/db/lockman/base.go

@@ -0,0 +1,51 @@
+package lockman
+
+import "context"
+
+type SBaseLockManager struct {
+	manager ILockManager
+}
+
+func NewBaseLockManger(m ILockManager) *SBaseLockManager {
+	return &SBaseLockManager{manager: m}
+}
+
+func (m *SBaseLockManager) LockClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
+	key := getClassKey(manager, projectId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseObject(ctx context.Context, model ILockedObject) {
+	key := getObjectKey(model)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseRawObject(ctx context.Context, resName string, resId string) {
+	key := getRawObjectKey(resName, resId)
+	m.manager.UnlockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.LockKey(ctx, key)
+}
+
+func (m *SBaseLockManager) ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
+	key := getJointObjectKey(model, model2)
+	m.manager.UnlockKey(ctx, key)
+}

pkg/cloudcommon/db/lockman/etcd.go

@@ -167,6 +167,7 @@ type SLockTableIndex struct {
 }
 
 type SEtcdLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[SLockTableIndex]*SEtcdLockRecord
 
@@ -203,6 +204,7 @@ func NewEtcdLockManager(config *SEtcdLockManagerConfig) (ILockManager, error) {
 		Value:  lockman,
 		Func:   atexit.ExitHandlerFunc(lockman.destroyAtExit),
 	})
+	lockman.SBaseLockManager = NewBaseLockManger(&lockman)
 	return &lockman, nil
 }
 

pkg/cloudcommon/db/lockman/inmemory.go

@@ -135,12 +135,17 @@ func (rec *SInMemoryLockRecord) unlockContext(ctx context.Context) (needClean bo
 }
 
 type SInMemoryLockManager struct {
+	*SBaseLockManager
 	tableLock *sync.Mutex
 	lockTable map[string]*SInMemoryLockRecord
 }
 
 func NewInMemoryLockManager() ILockManager {
-	lockMan := SInMemoryLockManager{tableLock: &sync.Mutex{}, lockTable: make(map[string]*SInMemoryLockRecord)}
+	lockMan := SInMemoryLockManager{
+		tableLock: &sync.Mutex{},
+		lockTable: make(map[string]*SInMemoryLockRecord),
+	}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }
 

pkg/cloudcommon/db/lockman/lockman.go

@@ -31,6 +31,15 @@ type ILockedObject interface {
 type ILockManager interface {
 	LockKey(ctx context.Context, key string)
 	UnlockKey(ctx context.Context, key string)
+
+	LockClass(ctx context.Context, manager ILockedClass, projectId string)
+	ReleaseClass(ctx context.Context, manager ILockedClass, projectId string)
+	LockObject(ctx context.Context, model ILockedObject)
+	ReleaseObject(ctx context.Context, model ILockedObject)
+	LockRawObject(ctx context.Context, resName string, resId string)
+	ReleaseRawObject(ctx context.Context, resName string, resId string)
+	LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
+	ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject)
 }
 
 func getClassKey(manager ILockedClass, projectId string) string {
@@ -64,41 +73,33 @@ func Init(man ILockManager) {
 }
 
 func LockClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockClass(ctx, manager, projectId)
 }
 
 func ReleaseClass(ctx context.Context, manager ILockedClass, projectId string) {
-	key := getClassKey(manager, projectId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseClass(ctx, manager, projectId)
 }
 
 func LockObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockObject(ctx, model)
 }
 
 func ReleaseObject(ctx context.Context, model ILockedObject) {
-	key := getObjectKey(model)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseObject(ctx, model)
 }
 
 func LockRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockRawObject(ctx, resName, resId)
 }
 
 func ReleaseRawObject(ctx context.Context, resName string, resId string) {
-	key := getRawObjectKey(resName, resId)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseRawObject(ctx, resName, resId)
 }
 
 func LockJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.LockKey(ctx, key)
+	_lockman.LockJointObject(ctx, model, model2)
 }
 
 func ReleaseJointObject(ctx context.Context, model ILockedObject, model2 ILockedObject) {
-	key := getJointObjectKey(model, model2)
-	_lockman.UnlockKey(ctx, key)
+	_lockman.ReleaseJointObject(ctx, model, model2)
 }

pkg/cloudcommon/db/lockman/noop.go

@@ -21,6 +21,7 @@ import (
 )
 
 type SNoopLockManager struct {
+	*SBaseLockManager
 }
 
 func (lockman *SNoopLockManager) LockKey(ctx context.Context, key string) {
@@ -33,5 +34,6 @@ func (lockman *SNoopLockManager) UnlockKey(ctx context.Context, key string) {
 
 func NewNoopLockManager() ILockManager {
 	lockMan := SNoopLockManager{}
+	lockMan.SBaseLockManager = NewBaseLockManger(&lockMan)
 	return &lockMan
 }

pkg/cloudcommon/db/managed.go

@@ -22,6 +22,7 @@ import (
 
 type IOwnerResourceBaseModel interface {
 	GetChangeOwnerCandidateDomainIds() []string
+	GetChangeOwnerRequiredDomainIds() []string
 }
 
 type IManagedResourceBase interface {

pkg/cloudcommon/db/modelbase.go

@@ -497,6 +497,13 @@ func (model *SModelBase) PerformAction(ctx context.Context, userCred mcclient.To
 	return nil, httperrors.NewActionNotFoundError("Action %s not found", action)
 }
 
+func (model *SModelBase) PreCheckPerformAction(
+	ctx context.Context, userCred mcclient.TokenCredential,
+	action string, query jsonutils.JSONObject, data jsonutils.JSONObject,
+) error {
+	return nil
+}
+
 // update hooks
 func (model *SModelBase) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool {
 	return false

pkg/cloudcommon/db/opslog.go

@@ -625,17 +625,11 @@ func (self *SOpsLogManager) FilterByOwner(q *sqlchemy.SQuery, ownerId mcclient.I
 			}
 		case rbacutils.ScopeProject:
 			if len(ownerId.GetProjectId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_tenant_id"), ownerId.GetProjectId()),
-					sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), ownerId.GetProjectId()))
 			}
 		case rbacutils.ScopeDomain:
 			if len(ownerId.GetProjectDomainId()) > 0 {
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.Equals(q.Field("owner_domain_id"), ownerId.GetProjectDomainId()),
-					sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()),
-				))
+				q = q.Filter(sqlchemy.Equals(q.Field("domain_id"), ownerId.GetProjectDomainId()))
 			}
 		default:
 			// systemScope, no filter

pkg/cloudcommon/db/quotas/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -481,7 +482,7 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) sendQuotaList(w http.ResponseWriter, quotaList []jsonutils.JSONObject) {
@@ -524,7 +525,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	manager.sendQuotaList(w, quotaList)
+	manager.sendQuotaList(w, sortQuotaByUsage(quotaList))
 }
 
 func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
@@ -616,3 +617,33 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 	}
 	return ret, nil
 }
+
+type tQuotaResultList []jsonutils.JSONObject
+
+func (a tQuotaResultList) Len() int           { return len(a) }
+func (a tQuotaResultList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a tQuotaResultList) Less(i, j int) bool { return usageRateOfQuota(a[i]) > usageRateOfQuota(a[j]) }
+
+func usageRateOfQuota(quota jsonutils.JSONObject) float32 {
+	maxRate := float32(0)
+	quotaMap, _ := quota.GetMap()
+	for k, v := range quotaMap {
+		usageK := fmt.Sprintf("usage.%s", k)
+		if usageV, ok := quotaMap[usageK]; ok {
+			intV, _ := v.Int()
+			if intV > 0 {
+				intUsageV, _ := usageV.Int()
+				rate := float32(intUsageV) / float32(intV)
+				if maxRate < rate {
+					maxRate = rate
+				}
+			}
+		}
+	}
+	return maxRate
+}
+
+func sortQuotaByUsage(quotaList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	sort.Sort(tQuotaResultList(quotaList))
+	return quotaList
+}

pkg/cloudcommon/db/quotas/interface.go

@@ -71,6 +71,7 @@ type IQuotaManager interface {
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
 	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)

pkg/cloudcommon/db/quotas/quotas.go

@@ -113,6 +113,17 @@ func (manager *SQuotaBaseManager) _cancelUsage(ctx context.Context, userCred mcc
 	return manager.changeUsage(ctx, userCred, usage, false)
 }
 
+func (manager *SQuotaBaseManager) addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	LockQuota(ctx, manager, usage)
+	defer ReleaseQuota(ctx, manager, usage)
+
+	return manager._addUsage(ctx, userCred, usage)
+}
+
+func (manager *SQuotaBaseManager) _addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	return manager.changeUsage(ctx, userCred, usage, true)
+}
+
 func (manager *SQuotaBaseManager) changeUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota, isAdd bool) error {
 	usages, err := manager.usageStore.GetParentQuotas(ctx, usage.GetKeys())
 	if err != nil {

pkg/cloudcommon/db/quotas/register.go

@@ -87,6 +87,20 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 	}
 }
 
+func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	for _, usage := range usages {
+		addUsage(ctx, userCred, usage.(IQuota))
+	}
+}
+
+func addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) {
+	manager := getQuotaManager(usage)
+	err := manager.addUsage(ctx, userCred, usage)
+	if err != nil {
+		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
+	}
+}
+
 func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
 	manager := getQuotaManager(request)
 	return manager.getQuotaCount(ctx, request, pendingKeys)