Merge pull request #7486 from wanyaoqi/automated-cherry-pick-of-#7484-upstream-release-3.2

Automated cherry pick of #7484: fix host info get sshkey

cmd/climc/shell/globalvpcs.go

@@ -15,6 +15,8 @@
 package shell
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -78,4 +80,30 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	R(&GlobalVpcShowOptions{}, "global-vpc-change-owner-candidate-domains", "Show candiate domains of a global vpc for changing owner", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
+		result, err := modules.GlobalVpcs.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type GlobalVpcChangeOwnerOptions struct {
+		ID            string `help:"ID or name of vpc" json:"-"`
+		ProjectDomain string `json:"project_domain" help:"target domain"`
+	}
+	R(&GlobalVpcChangeOwnerOptions{}, "global-vpc-change-owner", "Change owner domain of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcChangeOwnerOptions) error {
+		if len(args.ProjectDomain) == 0 {
+			return fmt.Errorf("empty project_domain")
+		}
+		params := jsonutils.Marshal(args)
+		ret, err := modules.GlobalVpcs.PerformAction(s, args.ID, "change-owner", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/images.go

@@ -252,7 +252,7 @@ func init() {
 	}
 
 	type ImageDeleteOptions struct {
-		ID                    []string `help:"Image ID or name"`
+		ID                    []string `help:"Image ID or name" json:"-"`
 		OverridePendingDelete *bool    `help:"Delete image directly instead of pending delete" short-token:"f"`
 	}
 	R(&ImageDeleteOptions{}, "image-delete", "Delete a image", func(s *mcclient.ClientSession, args *ImageDeleteOptions) error {

cmd/climc/shell/secgrouprules.go

@@ -25,14 +25,14 @@ import (
 func init() {
 	type SecGroupRulesListOptions struct {
 		options.BaseListOptions
-		Secgroup     string `help:"Secgroup ID or Name"`
-		SecgroupName string `help:"Search rules by fuzzy secgroup name"`
-		Project      string `help:"Filter rules by project"`
-		Direction    string `help:"filter Direction of rule" choices:"in|out"`
-		Protocol     string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Action       string `help:"filter Actin of rule" choices:"allow|deny"`
-		Ports        string `help:"filter Ports of rule"`
-		Ip           string `help:"filter cidr of rule"`
+		Secgroup     string   `help:"Secgroup ID or Name"`
+		SecgroupName string   `help:"Search rules by fuzzy secgroup name"`
+		Projects     []string `help:"Filter rules by project"`
+		Direction    string   `help:"filter Direction of rule" choices:"in|out"`
+		Protocol     string   `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
+		Action       string   `help:"filter Actin of rule" choices:"allow|deny"`
+		Ports        string   `help:"filter Ports of rule"`
+		Ip           string   `help:"filter cidr of rule"`
 	}
 
 	R(&SecGroupRulesListOptions{}, "secgroup-rule-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupRulesListOptions) error {

cmd/googlecli/main.go

@@ -18,17 +18,21 @@ import (
 	"fmt"
 	"os"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
 	Debug        bool   `help:"debug mode"`
 	Help         bool   `help:"Show help"`
+	AuthFile     string `help:"google cloud auth json file path" default:"$GOOGLE_AUTH_FILE"`
 	ClientEmail  string `help:"Client email" default:"$GOOGLE_CLIENT_EMAIL"`
 	ProjectID    string `help:"Project ID" default:"$GOOGLE_PROJECT_ID"`
 	PrivateKeyID string `help:"Private Key ID" default:"$GOOGLE_PRIVATE_KEY_ID"`
@@ -78,6 +82,20 @@ func showErrorAndExit(e error) {
 }
 
 func newClient(options *BaseOptions) (*google.SRegion, error) {
+	if len(options.AuthFile) > 0 {
+		jsonStr, err := fileutils2.FileGetContents(options.AuthFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "FileGetContents")
+		}
+		jsonCfg, err := jsonutils.ParseString(jsonStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.ParseString")
+		}
+		options.ClientEmail, _ = jsonCfg.GetString("client_email")
+		options.PrivateKeyID, _ = jsonCfg.GetString("private_key_id")
+		options.PrivateKey, _ = jsonCfg.GetString("private_key")
+		options.ProjectID, _ = jsonCfg.GetString("project_id")
+	}
 	if len(options.ClientEmail) == 0 {
 		return nil, fmt.Errorf("Missing ClientEmail")
 	}

go.mod

@@ -117,6 +117,7 @@ require (
 	google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb
 	google.golang.org/grpc v1.26.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/fatih/set.v0 v0.2.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
@@ -132,7 +133,7 @@ require (
 	yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 	yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11
-	yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8
+	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 
 replace (

go.sum

@@ -1013,6 +1013,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fatih/set.v0 v0.2.1 h1:Xvyyp7LXu34P0ROhCyfXkmQCAoOUKb1E2JS9I7SE5CY=
+gopkg.in/fatih/set.v0 v0.2.1/go.mod h1:5eLWEndGL4zGGemXWrKuts+wTJR0y+w+auqUJZbmyBg=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
@@ -1123,5 +1125,5 @@ yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11 h1:srrssomGJLh4OaDaY/1so+2FUBjZxdwQx/1pZ6403YU=
 yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8 h1:35Q2ce/gI2cuVhsuLS7WsL2K9SxwMEtWTopt/99iYsk=
-yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/misc.go

@@ -28,6 +28,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -37,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -57,6 +59,12 @@ const (
 	BATCH_HOST_REGISTER_QUANTITY_LIMITATION = 1000
 )
 
+var (
+	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
+	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+)
+
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
 	token := AppContextToken(ctx)
 	session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion)
@@ -170,20 +178,43 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	// check header line
+	titlesOk := false
+	for _, t := range [][]string{BatchHostRegisterTemplate, BatchHostISORegisterTemplate, BatchHostPXERegisterTemplate} {
+		if len(t) == len(rows[0]) {
+			for _, title := range rows[0] {
+				if !utils.IsInStringArray(title, t) {
+					break
+				}
+			}
+
+			titlesOk = true
+		}
+	}
+
+	if !titlesOk {
+		httperrors.InputParameterError(w, "template file is invalid.please check.")
+		return
+	}
+
 	paramKeys := []string{}
-	for _, title := range rows[0] {
+	i1 := -1
+	i2 := -1
+	for i, title := range rows[0] {
 		switch title {
 		case HOST_MAC:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
 		case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL:
+			i1 = i
 			paramKeys = append(paramKeys, "ipmi_ip_addr")
 		case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_username")
 		case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL:
 			paramKeys = append(paramKeys, "ipmi_password")
 		case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL:
+			i2 = i
 			paramKeys = append(paramKeys, "access_ip")
 		default:
 			e := httperrors.NewInternalServerError("empty file content")
@@ -199,8 +230,33 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 		return
 	}
 
+	ips := []string{}
 	hosts := bytes.Buffer{}
 	for _, row := range rows[1:] {
+		var e *httputils.JSONClientError
+		if i1 >= 0 && len(row[i1]) > 0 {
+			i1Ip := fmt.Sprintf("%d-%s", i1, row[i1])
+			if utils.IsInStringArray(i1Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i1])
+			} else {
+				ips = append(ips, i1Ip)
+			}
+		}
+
+		if i2 >= 0 && len(row[i2]) > 0 {
+			i2Ip := fmt.Sprintf("%d-%s", i2, row[i2])
+			if utils.IsInStringArray(i2Ip, ips) {
+				e = httperrors.NewDuplicateIdError("ip", row[i2])
+			} else {
+				ips = append(ips, i2Ip)
+			}
+		}
+
+		if e != nil {
+			httperrors.JsonClientError(w, e)
+			return
+		}
+
 		hosts.WriteString(strings.Join(row, ",") + "\n")
 	}
 
@@ -375,21 +431,21 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 	var content bytes.Buffer
 	switch template {
 	case "BatchHostRegister":
-		records := [][]string{{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}}
+		records := [][]string{BatchHostRegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostISORegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}}
+		records := [][]string{BatchHostISORegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")
 			return
 		}
 	case "BatchHostPXERegister":
-		records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}}
+		records := [][]string{BatchHostPXERegisterTemplate}
 		content, err = writeXlsx("hosts", records)
 		if err != nil {
 			httperrors.InternalServerError(w, "internal server error")

pkg/apis/compute/cloudaccount.go

@@ -180,9 +180,9 @@ type CloudaccountCreateInput struct {
 	// 自动同步间隔时间
 	SyncIntervalSeconds int `json:"sync_interval_seconds"`
 
-	// 自动根据云上项目或订阅创建本地项目
+	// 自动根据云上项目或订阅创建本地项目, OpenStack此参数为true
 	// default: false
-	AutoCreateProject bool `json:"auto_create_project"`
+	AutoCreateProject *bool `json:"auto_create_project"`
 
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONDict `json:"options"`

pkg/apis/compute/cloudprovider.go

@@ -251,3 +251,23 @@ type CloudproviderUpdateInput struct {
 
 type CloudproviderCreateInput struct {
 }
+
+type CloudproviderGetStorageClassInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetStorageClassOutput struct {
+	// 对象存储存储类型
+	StorageClasses []string `json:"storage_classes"`
+}
+
+type CloudproviderGetCannedAclInput struct {
+	CloudregionResourceInput
+}
+
+type CloudproviderGetCannedAclOutput struct {
+	// Bucket支持的预置ACL列表
+	BucketCannedAcls []string `json:"bucket_canned_acls"`
+	// Object支持的预置ACL列表
+	ObjectCannedAcls []string `json:"object_canned_acls"`
+}

pkg/apis/compute/dbinstance.go

@@ -101,7 +101,7 @@ type DBInstanceCreateInput struct {
 	// | -----		| ------	| --- |
 	// | 华为云		|ha, single, replica| |
 	// | 阿里云		|basic, high_availability, always_on, finance||
-	// | Google		|SECOND_GEN | FIRST_GEN 目前谷歌已弃用|
+	// | Google		|Zonal, Regional | |
 	// 翻译:
 	// basic: 基础版
 	// high_availability: 高可用
@@ -110,7 +110,8 @@ type DBInstanceCreateInput struct {
 	// ha: 高可用
 	// single: 单机
 	// replica: 只读
-	// SECNOD_GEN: 第二代
+	// Zonal: 单区域
+	// Regional: 区域级
 	// required: true
 	Category string `json:"category"`
 
@@ -170,9 +171,22 @@ type SDBInstanceChangeConfigInput struct {
 type SDBInstanceRecoveryConfigInput struct {
 	apis.Meta
 
-	DBInstancebackup   string
-	DBInstancebackupId string            `json:"dbinstancebackup_id"`
-	Databases          map[string]string `json:"databases,allowempty"`
+	// swagger:ignore
+	DBInstancebackup string `json:"dbinstancebackup" "yunion:deprecated-by":"dbinstancebackup_id"`
+
+	// 备份Id
+	//
+	//
+	// | 平台		| 支持引擎								| 说明		|
+	// | -----		| ------								| ---		|
+	// | 华为云		|MySQL, SQL Server						| 仅SQL Server支持恢复到当前实例			|
+	// | 阿里云		|MySQL, SQL Server						| MySQL要求必须开启单库单表恢复功能 并且只能是MySQL 8.0 高可用版（本地SSD盘）MySQL 5.7 高可用版（本地SSD盘）或MySQL 5.6 高可用版, MySQL仅支持恢复到当前实例|
+	// | Google		|MySQL, PostgreSQL, SQL Server			| PostgreSQL备份恢复时，要求实例不能有副本			|
+	DBInstancebackupId string `json:"dbinstancebackup_id"`
+
+	// 数据库信息, 例如 {"src":"dest"} 是将备份中的src数据库恢复到目标实例的dest数据库中, 阿里云此参数为必传
+	// example: {"sdb1":"ddb1"}
+	Databases map[string]string `json:"databases,allowempty"`
 }
 
 type DBInstanceListInput struct {

pkg/apis/compute/dbinstance_account.go

@@ -70,13 +70,13 @@ type SDBInstanceSetPrivilegesInput struct {
 
 type DBInstancePrivilege struct {
 	// 数据库名称
-	Database string
+	Database string `json:"database"`
 	// 账号名称
-	Account string
+	Account string `json:"account"`
 	// 数据库Id
-	DBInstancedatabaseId string
+	DBInstancedatabaseId string `json:"dbinstancedatabase_id"`
 	// 权限
-	Privileges string
+	Privileges string `json:"privileges"`
 }
 
 type DBInstanceAccountDetails struct {

pkg/apis/compute/dbinstance_const.go

@@ -16,19 +16,20 @@ package compute
 
 const (
 	//实例状态
-	DBINSTANCE_DEPLOYING      = "deploying"  //部署中
-	DBINSTANCE_RUNNING        = "running"    //运行中
-	DBINSTANCE_REBOOTING      = "rebooting"  //重启中
-	DBINSTANCE_MIGRATING      = "migrating"  //迁移中
-	DBINSTANCE_BACKING_UP     = "backing_up" //备份中
-	DBINSTANCE_RESTORING      = "restoring"  //备份恢复中
-	DBINSTANCE_RESTORE_FAILED = "restore_failed"
-	DBINSTANCE_IMPORTING      = "importing"     //数据导入中
-	DBINSTANCE_CLONING        = "cloning"       //克隆中
-	DBINSTANCE_DELETING       = "deleting"      //删除中
-	DBINSTANCE_DELETE_FAILED  = "delete_failed" //删除失败
-	DBINSTANCE_MAINTENANCE    = "maintenance"   //维护中
-	DBINSTANCE_UNKNOWN        = "unknown"
+	DBINSTANCE_DEPLOYING         = "deploying"         //部署中
+	DBINSTANCE_RUNNING           = "running"           //运行中
+	DBINSTANCE_REBOOTING         = "rebooting"         //重启中
+	DBINSTANCE_MIGRATING         = "migrating"         //迁移中
+	DBINSTANCE_BACKING_UP        = "backing_up"        //备份中
+	DBINSTANCE_BACKING_UP_FAILED = "backing_up_failed" //备份失败
+	DBINSTANCE_RESTORING         = "restoring"         //备份恢复中
+	DBINSTANCE_RESTORE_FAILED    = "restore_failed"
+	DBINSTANCE_IMPORTING         = "importing"     //数据导入中
+	DBINSTANCE_CLONING           = "cloning"       //克隆中
+	DBINSTANCE_DELETING          = "deleting"      //删除中
+	DBINSTANCE_DELETE_FAILED     = "delete_failed" //删除失败
+	DBINSTANCE_MAINTENANCE       = "maintenance"   //维护中
+	DBINSTANCE_UNKNOWN           = "unknown"
 
 	DBINSTANCE_CHANGE_CONFIG        = "change_config"        //调整配置
 	DBINSTANCE_CHANGE_CONFIG_FAILED = "change_config_failed" //调整配置失败

pkg/apis/compute/geo_input.go

@@ -64,6 +64,23 @@ type ZonalFilterListBase struct {
 	OrderByZone string `json:"order_by_zone"`
 }
 
+func (input ZonalFilterListBase) ZoneList() []string {
+	zoneStr := input.Zone
+	if len(zoneStr) > 0 {
+		input.Zones = append(input.Zones, zoneStr)
+	}
+	return input.Zones
+}
+
+func (input ZonalFilterListBase) FirstZone() string {
+	if len(input.Zone) > 0 {
+		return input.Zone
+	}
+	if len(input.Zones) > 0 {
+		return input.Zones[0]
+	}
+	return ""
+}
 func (input ZonalFilterListInput) ZoneList() []string {
 	zoneStr := input.Zone
 	if len(zoneStr) > 0 {

pkg/apis/compute/guesttemplate.go

@@ -57,6 +57,7 @@ type GuestTemplateDetails struct {
 
 	Secgroups []string `json:"secgroups"`
 	Zone      string   `json:"zone"`
+	ZoneId    string   `json:"zone_id"`
 	Brand     string   `json:"brand"`
 
 	ConfigInfo GuestTemplateConfigInfo `json:"config_info"`
@@ -71,9 +72,7 @@ type GuestTemplateListInput struct {
 }
 
 type GuestTemplateConfigInfo struct {
-	Region               string                 `json:"region"`
-	Zone                 string                 `json:"zone"`
-	Hypervisor           string                 `json:"hypervisor"`
+	Metadata             map[string]string      `json:"metadata"`
 	Secgroup             string                 `json:"secgroup"`
 	Sku                  GuestTemplateSku       `json:"sku"`
 	Disks                []GuestTemplateDisk    `json:"disks"`

pkg/apis/compute/host.go

@@ -138,7 +138,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率

pkg/apis/compute/scaling_group.go

@@ -134,9 +134,15 @@ type ScalingGroupDetails struct {
 	// example: OneCloud
 	Brand string `json:"brand"`
 
-	// description: 网络ID
-	// example: net-12345
-	Networks []string `json:"networks"`
+	// description: 网络信息
+	Networks []ScalingGroupNetwork `json:"networks"`
+}
+
+type ScalingGroupNetwork struct {
+	Id           string `json:"id"`
+	Name         string `json:"name"`
+	GuestIpStart string `json:"guest_ip_start"`
+	GuestIpEnd   string `json:"guest_ip_end"`
 }
 
 type ScalingGroupResourceInfo struct {

pkg/apis/compute/secgroup.go

@@ -174,6 +174,8 @@ type SecurityGroupRuleListInput struct {
 	apis.ResourceBaseListInput
 	SecgroupFilterListInput
 
+	Projects []string `json:"projects"`
+
 	// 以direction字段过滤安全组规则
 	Direction string `json:"direction"`
 	// 以action字段过滤安全组规则
@@ -196,8 +198,6 @@ type SecgroupResourceInput struct {
 
 	// 模糊匹配安全组规则名称
 	SecgroupName string `json:"secgroup_name"`
-
-	apis.ProjectizedResourceInput
 }
 
 type SecgroupFilterListInput struct {

pkg/apis/compute/storage_const.go

@@ -127,7 +127,7 @@ var (
 		STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS,
 	}
 
-	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA}
 
 	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
 

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -249,11 +249,17 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme
 			return fmt.Errorf("getLastArray: %v", err)
 		}
 		cmds := []string{}
-		for _, sz := range conf.Size[1:] {
+		restSize := conf.Size[1:]
+		for idx, sz := range restSize {
+			isLast := idx == len(restSize)-1
+			sizeStr := fmt.Sprintf("size=%d", sz)
+			if isLast {
+				sizeStr = "size=max"
+			}
 			args = []string{"controller", fmt.Sprintf("slot=%d", adapter.index),
 				"array", array, "create", "type=ld",
 				fmt.Sprintf("raid=%s", level),
-				fmt.Sprintf("size=%d", sz),
+				sizeStr,
 			}
 			args = append(args, params...)
 			cmds = append(cmds, GetCommand(args...))

pkg/cloudcommon/db/domain.go

@@ -77,6 +77,10 @@ func (model *SDomainizedResourceBase) GetChangeOwnerCandidateDomainIds() []strin
 	return nil
 }
 
+func (model *SDomainizedResourceBase) GetChangeOwnerRequiredDomainIds() []string {
+	return nil
+}
+
 func ValidateCreateDomainId(domainId string) error {
 	if !consts.GetNonDefaultDomainProjects() && domainId != identity.DEFAULT_DOMAIN_ID {
 		return httperrors.NewForbiddenError("project in non-default domain is prohibited")

pkg/cloudcommon/db/domainresource.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -133,6 +134,9 @@ func (model *SDomainLevelResourceBase) AllowPerformChangeOwner(ctx context.Conte
 }
 
 func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformChangeDomainOwnerInput) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.GetIStandaloneModel().IsShared() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot change owner of shared resource")
 	}
@@ -162,6 +166,11 @@ func (model *SDomainLevelResourceBase) PerformChangeOwner(ctx context.Context, u
 	if len(candidates) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), candidates) {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner candidate list")
 	}
+	requires := model.GetIDomainLevelModel().GetChangeOwnerRequiredDomainIds()
+	log.Debugf("%s required domains: %s", model.Keyword(), requires)
+	if len(requires) > 0 && !utils.IsInStringArray(ownerId.GetProjectDomainId(), requires) {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "target domain not in change owner required list")
+	}
 
 	if !IsAdminAllowPerform(userCred, model, "change-owner") {
 		return nil, errors.Wrap(httperrors.ErrNotSufficientPrivilege, "require system privileges")

pkg/cloudcommon/db/infraresource.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -200,6 +201,9 @@ func (model *SInfrasResourceBase) PerformChangeOwner(
 	query jsonutils.JSONObject,
 	input apis.PerformChangeDomainOwnerInput,
 ) (jsonutils.JSONObject, error) {
+	if !consts.GetNonDefaultDomainProjects() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "not allow to change owner of domain resource if non_default_domain_projects is turned off")
+	}
 	if model.IsShared() {
 		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "cannot change owner when shared!")
 	}

pkg/cloudcommon/db/managed.go

@@ -22,6 +22,7 @@ import (
 
 type IOwnerResourceBaseModel interface {
 	GetChangeOwnerCandidateDomainIds() []string
+	GetChangeOwnerRequiredDomainIds() []string
 }
 
 type IManagedResourceBase interface {

pkg/cloudcommon/db/quotas/interface.go

@@ -71,6 +71,7 @@ type IQuotaManager interface {
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
 	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)

pkg/cloudcommon/db/quotas/quotas.go

@@ -113,6 +113,17 @@ func (manager *SQuotaBaseManager) _cancelUsage(ctx context.Context, userCred mcc
 	return manager.changeUsage(ctx, userCred, usage, false)
 }
 
+func (manager *SQuotaBaseManager) addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	LockQuota(ctx, manager, usage)
+	defer ReleaseQuota(ctx, manager, usage)
+
+	return manager._addUsage(ctx, userCred, usage)
+}
+
+func (manager *SQuotaBaseManager) _addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error {
+	return manager.changeUsage(ctx, userCred, usage, true)
+}
+
 func (manager *SQuotaBaseManager) changeUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota, isAdd bool) error {
 	usages, err := manager.usageStore.GetParentQuotas(ctx, usage.GetKeys())
 	if err != nil {

pkg/cloudcommon/db/quotas/register.go

@@ -87,6 +87,20 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 	}
 }
 
+func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	for _, usage := range usages {
+		addUsage(ctx, userCred, usage.(IQuota))
+	}
+}
+
+func addUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) {
+	manager := getQuotaManager(usage)
+	err := manager.addUsage(ctx, userCred, usage)
+	if err != nil {
+		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
+	}
+}
+
 func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
 	manager := getQuotaManager(request)
 	return manager.getQuotaCount(ctx, request, pendingKeys)

pkg/cloudcommon/policy/defaults.go

@@ -29,6 +29,11 @@ var (
 					Action:   PolicyActionPerform,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Resource: "metadatas",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 		{

pkg/cloudprovider/cloudprovider.go

@@ -190,6 +190,8 @@ type ICloudProviderFactory interface {
 	IsCloudeventRegional() bool
 	GetMaxCloudEventSyncDays() int
 	GetMaxCloudEventKeepDays() int
+
+	IsNeedForceAutoCreateProject() bool
 }
 
 type ICloudProvider interface {
@@ -215,6 +217,8 @@ type ICloudProvider interface {
 	GetCloudRegionExternalIdPrefix() string
 
 	GetStorageClasses(regionId string) []string
+	GetBucketCannedAcls(regionId string) []string
+	GetObjectCannedAcls(regionId string) []string
 
 	GetCapabilities() []string
 	GetICloudQuotas() ([]ICloudQuota, error)
@@ -408,6 +412,10 @@ func (factory *baseProviderFactory) GetMaxCloudEventKeepDays() int {
 	return 7
 }
 
+func (factory *baseProviderFactory) IsNeedForceAutoCreateProject() bool {
+	return false
+}
+
 type SPremiseBaseProviderFactory struct {
 	baseProviderFactory
 }

pkg/cloudprovider/dbinstance.go

@@ -88,6 +88,7 @@ type SDBInstanceBackupCreateConfig struct {
 }
 
 type SDBInstanceRecoveryConfig struct {
-	BackupId  string
-	Databases map[string]string
+	BackupId                   string
+	Databases                  map[string]string
+	OriginDBInstanceExternalId string
 }

pkg/cloudprovider/fakeregion.go

@@ -97,7 +97,7 @@ func (region *SFakeOnPremiseRegion) GetISecurityGroupById(id string) (ICloudSecu
 	return nil, ErrNotSupported
 }
 
-func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error) {
+func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error) {
 	return nil, ErrNotSupported
 }
 

pkg/cloudprovider/network.go

@@ -0,0 +1,22 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudprovider
+
+type SNetworkCreateOptions struct {
+	Name      string
+	Desc      string
+	ProjectId string
+	Cidr      string
+}

pkg/cloudprovider/resources.go

@@ -23,6 +23,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type ICloudResource interface {
@@ -70,7 +71,7 @@ type ICloudRegion interface {
 	GetIDiskById(id string) (ICloudDisk, error)
 
 	GetISecurityGroupById(secgroupId string) (ICloudSecurityGroup, error)
-	GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error)
+	GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error)
 	CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error)
 
 	CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error)
@@ -342,7 +343,7 @@ type ICloudEIP interface {
 }
 
 type ICloudSecurityGroup interface {
-	ICloudResource
+	IVirtualResource
 
 	GetDescription() string
 	GetRules() ([]SecurityRule, error)
@@ -450,7 +451,7 @@ type ICloudWire interface {
 
 	GetINetworkById(netid string) (ICloudNetwork, error)
 
-	CreateINetwork(name string, cidr string, desc string) (ICloudNetwork, error)
+	CreateINetwork(opts *SNetworkCreateOptions) (ICloudNetwork, error)
 }
 
 type ICloudNetwork interface {
@@ -463,8 +464,11 @@ type ICloudNetwork interface {
 	GetIpMask() int8
 	GetGateway() string
 	GetServerType() string
-	// GetIsPublic() bool
-	// GetPublicScope() rbacutils.TRbacScope
+	//GetIsPublic() bool
+	// 仅私有云有用，公有云无效
+	// 1. scope = none 非共享, network仅会属于一个项目,并且私有
+	// 2. scope = system 系统共享 云账号共享会跟随云账号共享，云账号非共享,会共享到network所在域
+	GetPublicScope() rbacutils.TRbacScope
 
 	Delete() error
 

pkg/cloudprovider/securitygroup.go

@@ -24,11 +24,18 @@ import (
 
 const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id"
 
+type SecurityGroupFilterOptions struct {
+	VpcId     string
+	Name      string
+	ProjectId string
+}
+
 type SecurityGroupCreateInput struct {
-	Name  string
-	Desc  string
-	VpcId string
-	Rules []secrules.SecurityRule
+	Name      string
+	Desc      string
+	VpcId     string
+	ProjectId string
+	Rules     []secrules.SecurityRule
 }
 
 type SecurityRule struct {

pkg/compute/guestdrivers/managedvirtual.go

@@ -321,7 +321,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context
 
 		secgroups := guest.GetSecgroups()
 		for i, secgroup := range secgroups {
-			externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup)
+			externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, desc.ProjectId)
 			if err != nil {
 				return errors.Wrap(err, "RequestSyncSecurityGroup")
 			}
@@ -976,10 +976,19 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncSecgroupsOnHost(ctx conte
 		return errors.Wrap(err, "GetSecurityGroupVpcId")
 	}
 
+	remoteProjectId := ""
+	provider := host.GetCloudprovider()
+	if provider != nil {
+		remoteProjectId, err = provider.SyncProject(ctx, task.GetUserCred(), guest.ProjectId)
+		if err != nil {
+			log.Errorf("failed to sync project %s for guest %s error: %v", guest.ProjectId, guest.Name, err)
+		}
+	}
+
 	secgroups := guest.GetSecgroups()
 	externalIds := []string{}
 	for _, secgroup := range secgroups {
-		externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup)
+		externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, remoteProjectId)
 		if err != nil {
 			return errors.Wrap(err, "RequestSyncSecurityGroup")
 		}

pkg/compute/guestdrivers/openstack.go

@@ -114,6 +114,10 @@ func (self *SOpenStackGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudpr
 	return true
 }
 
+func (self *SOpenStackGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
+	return true
+}
+
 func (self *SOpenStackGuestDriver) IsNeedRestartForResetLoginInfo() bool {
 	return false
 }

pkg/compute/models/buckets.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -420,6 +421,16 @@ func (manager *SBucketManager) ValidateCreateData(
 		return input, httperrors.NewInputParameterError("invalid bucket name %s: %s", input.Name, err)
 	}
 
+	if len(input.StorageClass) > 0 {
+		driver, err := managerV.GetProvider()
+		if err != nil {
+			return input, errors.Wrap(err, "GetProvider")
+		}
+		if !utils.IsInStringArray(input.StorageClass, driver.GetStorageClasses(cloudRegionV.Id)) {
+			return input, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", input.StorageClass)
+		}
+	}
+
 	quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, cloudRegionV, managerV)
 	pendingUsage := SRegionQuota{Bucket: 1}
 	pendingUsage.SetKeys(quotaKeys)
@@ -956,14 +967,17 @@ func (bucket *SBucket) PerformUpload(
 		return nil, httperrors.NewInputParameterError("Content-Length negative %d", sizeBytes)
 	}
 	storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
+	driver, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
+	}
+	if len(storageClass) > 0 && !utils.IsInStringArray(storageClass, driver.GetStorageClasses(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid storage class %s", storageClass)
+	}
+
 	aclStr := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)
-	if len(aclStr) > 0 {
-		switch cloudprovider.TBucketACLType(aclStr) {
-		case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
-			// do nothing
-		default:
-			return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
-		}
+	if len(aclStr) > 0 && !utils.IsInStringArray(aclStr, driver.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "invalid acl %s", aclStr)
 	}
 
 	inc := cloudprovider.SBucketStats{}
@@ -1038,15 +1052,23 @@ func (bucket *SBucket) PerformAcl(
 ) (jsonutils.JSONObject, error) {
 	err := input.Validate()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "ValidateInput")
+	}
+
+	provider, err := bucket.GetDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetDriver")
 	}
 
 	iBucket, objects, err := bucket.processObjectsActionInput(input.BucketObjectsActionInput)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "processObjectsActionInput")
 	}
 
 	if len(objects) == 0 {
+		if !utils.IsInStringArray(string(input.Acl), provider.GetBucketCannedAcls(bucket.CloudregionId)) {
+			return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported bucket canned acl %s", input.Acl)
+		}
 		err = iBucket.SetAcl(input.Acl)
 		if err != nil {
 			return nil, httperrors.NewInternalServerError("setAcl error %s", err)
@@ -1059,6 +1081,10 @@ func (bucket *SBucket) PerformAcl(
 		return nil, nil
 	}
 
+	if !utils.IsInStringArray(string(input.Acl), provider.GetObjectCannedAcls(bucket.CloudregionId)) {
+		return nil, errors.Wrapf(httperrors.ErrInputParameter, "unsupported object canned acl %s", input.Acl)
+	}
+
 	errs := make([]error, 0)
 	for _, object := range objects {
 		err := object.SetAcl(input.Acl)

pkg/compute/models/cloudaccounts.go

@@ -35,7 +35,6 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
@@ -47,7 +46,6 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -342,17 +340,37 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		return input, err
 	}
 
-	if len(input.Project) > 0 {
-		_, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
-		if err != nil {
-			return input, errors.Wrap(err, "db.ValidateProjectizedResourceInput")
-		}
-	}
-
 	if !cloudprovider.IsSupported(input.Provider) {
 		return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider)
 	}
 	providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider)
+
+	forceAutoCreateProject := providerDriver.IsNeedForceAutoCreateProject()
+	if forceAutoCreateProject {
+		input.AutoCreateProject = &forceAutoCreateProject
+	}
+
+	if len(input.Project) > 0 {
+		var proj *db.STenant
+		proj, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
+		if err != nil {
+			return input, errors.Wrap(err, "db.ValidateProjectizedResourceInput")
+		}
+		if proj.DomainId != ownerId.GetProjectDomainId() {
+			return input, httperrors.NewInputParameterError("Project %s(%s) not belong to domain %s(%s)", proj.Name, proj.Id, ownerId.GetProjectDomain(), ownerId.GetProjectDomainId())
+		}
+		if input.AutoCreateProject != nil && *input.AutoCreateProject {
+			log.Warningf("project_id and auto_create_project should not be turned on at the same time")
+		}
+		if !forceAutoCreateProject {
+			input.AutoCreateProject = nil
+		}
+	} else if input.AutoCreateProject == nil || !*input.AutoCreateProject {
+		log.Warningf("auto_create_project is off while no project_id specified")
+		createProject := true
+		input.AutoCreateProject = &createProject
+	}
+
 	input.SCloudaccount, err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, input.SCloudaccountCredential)
 	if err != nil {
 		return input, err
@@ -430,20 +448,6 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		input.SyncIntervalSeconds = options.Options.MinimalSyncIntervalSeconds
 	}
 
-	if !input.AutoCreateProject {
-		if userCred.GetProjectDomainId() != ownerId.GetProjectDomainId() {
-			s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
-			params := jsonutils.Marshal(map[string]string{"domain_id": ownerId.GetProjectDomainId()})
-			tenants, err := modules.Projects.List(s, params)
-			if err != nil {
-				return input, err
-			}
-			if tenants.Total == 0 {
-				return input, httperrors.NewInputParameterError("There is no projects under the domain %s", ownerId.GetProjectDomainId())
-			}
-		}
-	}
-
 	input.EnabledStatusInfrasResourceBaseCreateInput, err = manager.SEnabledStatusInfrasResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusInfrasResourceBaseCreateInput)
 	if err != nil {
 		return input, err
@@ -1899,9 +1903,11 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre
 	defer cloudaccountPendingSyncsMutex.Unlock()
 	if _, ok := cloudaccountPendingSyncs[account.Id]; ok {
 		if waitChan != nil {
-			// an active cloudaccount sync task is running, return with conflict error
-			log.Errorf("an active cloudaccount sync task is running, early return with conflict error")
-			waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs")
+			go func() {
+				// an active cloudaccount sync task is running, return with conflict error
+				log.Errorf("an active cloudaccount sync task is running, early return with conflict error")
+				waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs")
+			}()
 		}
 		return
 	}

pkg/compute/models/cloudproviders.go

@@ -1563,33 +1563,56 @@ func (provider *SCloudprovider) AllowGetDetailsStorageClasses(
 func (provider *SCloudprovider) GetDetailsStorageClasses(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
-	query jsonutils.JSONObject,
-) (jsonutils.JSONObject, error) {
+	input api.CloudproviderGetStorageClassInput,
+) (api.CloudproviderGetStorageClassOutput, error) {
+	output := api.CloudproviderGetStorageClassOutput{}
 	driver, err := provider.GetProvider()
 	if err != nil {
-		return nil, httperrors.NewInternalServerError("fail to get provider driver %s", err)
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
 	}
-	extId := ""
-	regionStr := jsonutils.GetAnyString(query, []string{"cloudregion", "cloudregion_id"})
-	if len(regionStr) > 0 {
-		regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr)
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
 		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
 		}
-		extId = regionObj.(*SCloudregion).GetExternalId()
 	}
 
-	sc := driver.GetStorageClasses(extId)
+	sc := driver.GetStorageClasses(input.Cloudregion)
 	if sc == nil {
-		return nil, httperrors.NewInternalServerError("storage classes not supported")
+		return output, httperrors.NewInternalServerError("storage classes not supported")
 	}
-	ret := jsonutils.NewDict()
-	ret.Add(jsonutils.NewStringArray(sc), "storage_classes")
-	return ret, nil
+	output.StorageClasses = sc
+	return output, nil
+}
+
+func (provider *SCloudprovider) AllowGetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) bool {
+	return db.IsAdminAllowGetSpec(userCred, provider, "canned-acls")
+}
+
+func (provider *SCloudprovider) GetDetailsCannedAcls(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	input api.CloudproviderGetCannedAclInput,
+) (api.CloudproviderGetCannedAclOutput, error) {
+	output := api.CloudproviderGetCannedAclOutput{}
+	driver, err := provider.GetProvider()
+	if err != nil {
+		return output, httperrors.NewInternalServerError("fail to get provider driver %s", err)
+	}
+	if len(input.Cloudregion) > 0 {
+		_, input.CloudregionResourceInput, err = ValidateCloudregionResourceInput(userCred, input.CloudregionResourceInput)
+		if err != nil {
+			return output, errors.Wrap(err, "ValidateCloudregionResourceInput")
+		}
+	}
+
+	output.BucketCannedAcls = driver.GetBucketCannedAcls(input.Cloudregion)
+	output.ObjectCannedAcls = driver.GetObjectCannedAcls(input.Cloudregion)
+	return output, nil
 }
 
 func (provider *SCloudprovider) getAccountShareInfo() apis.SAccountShareInfo {

pkg/compute/models/cloudsync.go

@@ -240,6 +240,10 @@ func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, sync
 			lockman.LockObject(ctx, &localVpcs[j])
 			defer lockman.ReleaseObject(ctx, &localVpcs[j])
 
+			if localVpcs[j].Deleted {
+				return
+			}
+
 			syncVpcWires(ctx, userCred, syncResults, provider, &localVpcs[j], remoteVpcs[j], syncRange)
 			if localRegion.GetDriver().IsSecurityGroupBelongVpc() || localRegion.GetDriver().IsSupportClassicSecurityGroup() || j == 0 { //有vpc属性的每次都同步,支持classic的vpc也同步，否则仅同步一次
 				syncVpcSecGroup(ctx, userCred, syncResults, provider, &localVpcs[j], remoteVpcs[j], syncRange)
@@ -311,6 +315,10 @@ func syncVpcNatgateways(ctx context.Context, userCred mcclient.TokenCredential,
 			lockman.LockObject(ctx, &localNatGateways[i])
 			defer lockman.ReleaseObject(ctx, &localNatGateways[i])
 
+			if localNatGateways[i].Deleted {
+				return
+			}
+
 			syncNatGatewayEips(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
 			syncNatDTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
 			syncNatSTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i])
@@ -392,6 +400,9 @@ func syncVpcWires(ctx context.Context, userCred mcclient.TokenCredential, syncRe
 			lockman.LockObject(ctx, &localWires[i])
 			defer lockman.ReleaseObject(ctx, &localWires[i])
 
+			if localWires[i].Deleted {
+				return
+			}
 			syncWireNetworks(ctx, userCred, syncResults, provider, &localWires[i], remoteWires[i], syncRange)
 		}()
 	}
@@ -446,6 +457,10 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy
 			lockman.LockObject(ctx, &localStorages[i])
 			defer lockman.ReleaseObject(ctx, &localStorages[i])
 
+			if localStorages[i].Deleted {
+				return
+			}
+
 			if !isInCache(storageCachePairs, localStorages[i].StoragecacheId) && !isInCache(newCacheIds, localStorages[i].StoragecacheId) {
 				cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i])
 				if cachePair.remote != nil && cachePair.local != nil {
@@ -531,6 +546,10 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR
 			lockman.LockObject(ctx, &localHosts[i])
 			defer lockman.ReleaseObject(ctx, &localHosts[i])
 
+			if localHosts[i].Deleted {
+				return
+			}
+
 			syncMetadata(ctx, userCred, &localHosts[i], remoteHosts[i])
 			newCachePairs = syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs)
 			syncHostWires(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i])
@@ -624,6 +643,10 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes
 			lockman.LockObject(ctx, syncVMPairs[i].Local)
 			defer lockman.ReleaseObject(ctx, syncVMPairs[i].Local)
 
+			if syncVMPairs[i].Local.Deleted || syncVMPairs[i].Local.PendingDeleted {
+				return
+			}
+
 			syncVMPeripherals(ctx, userCred, syncVMPairs[i].Local, syncVMPairs[i].Remote, localHost, provider, driver)
 			// syncMetadata(ctx, userCred, syncVMPairs[i].Local, syncVMPairs[i].Remote)
 			// syncVMNics(ctx, userCred, provider, localHost, syncVMPairs[i].Local, syncVMPairs[i].Remote)
@@ -773,6 +796,10 @@ func syncRegionDBInstances(ctx context.Context, userCred mcclient.TokenCredentia
 			lockman.LockObject(ctx, &localInstances[i])
 			defer lockman.ReleaseObject(ctx, &localInstances[i])
 
+			if localInstances[i].Deleted || localInstances[i].PendingDeleted {
+				return
+			}
+
 			syncDBInstanceNetwork(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
 			syncDBInstanceParameters(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
 			syncDBInstanceDatabases(ctx, userCred, syncResults, &localInstances[i], remoteInstances[i])
@@ -880,6 +907,10 @@ func syncDBInstanceAccounts(ctx context.Context, userCred mcclient.TokenCredenti
 			lockman.LockObject(ctx, &localAccounts[i])
 			defer lockman.ReleaseObject(ctx, &localAccounts[i])
 
+			if localAccounts[i].Deleted {
+				return
+			}
+
 			syncDBInstanceAccountPrivileges(ctx, userCred, syncResults, &localAccounts[i], remoteAccounts[i])
 
 		}()
@@ -963,6 +994,10 @@ func syncRegionNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCre
 			lockman.LockObject(ctx, &localInterfaces[i])
 			defer lockman.ReleaseObject(ctx, &localInterfaces[i])
 
+			if localInterfaces[i].Deleted {
+				return
+			}
+
 			syncInterfaceAddresses(ctx, userCred, &localInterfaces[i], remoteInterfaces[i])
 		}()
 	}
@@ -1141,15 +1176,17 @@ func syncOnPremiseCloudProviderInfo(
 		}
 	}
 
-	log.Debugf("storageCachePairs count %d", len(storageCachePairs))
-	for i := range storageCachePairs {
-		// alway sync on-premise cached images
-		// if storageCachePairs[i].isNew || syncRange.DeepSync {
-		result := storageCachePairs[i].syncCloudImages(ctx, userCred)
-		syncResults.Add(StoragecachedimageManager, result)
-		msg := result.Result()
-		log.Infof("syncCloudImages result: %s", msg)
-		// }
+	if cloudprovider.IsSupportCompute(driver) {
+		log.Debugf("storageCachePairs count %d", len(storageCachePairs))
+		for i := range storageCachePairs {
+			// alway sync on-premise cached images
+			// if storageCachePairs[i].isNew || syncRange.DeepSync {
+			result := storageCachePairs[i].syncCloudImages(ctx, userCred)
+			syncResults.Add(StoragecachedimageManager, result)
+			msg := result.Result()
+			log.Infof("syncCloudImages result: %s", msg)
+			// }
+		}
 	}
 
 	return nil

pkg/compute/models/dbinstance_backups.go

@@ -259,11 +259,16 @@ func (self *SDBInstanceBackup) PostCreate(ctx context.Context, userCred mcclient
 }
 
 func (self *SDBInstanceBackup) StartDBInstanceBackupCreateTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error {
-	self.SetStatus(userCred, api.DBINSTANCE_BACKUP_CREATING, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "DBInstanceBackupCreateTask", self, userCred, data, parentTaskId, "", nil)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "NewTask")
 	}
+	instance, err := self.GetDBInstance()
+	if err != nil {
+		return errors.Wrap(err, "GetDBInstance")
+	}
+	instance.SetStatus(userCred, api.DBINSTANCE_BACKING_UP, "")
+	self.SetStatus(userCred, api.DBINSTANCE_BACKUP_CREATING, "")
 	task.ScheduleRun(nil)
 	return nil
 }

pkg/compute/models/dbinstances.go

@@ -675,22 +675,19 @@ func (self *SDBInstance) AllowPerformRecovery(ctx context.Context, userCred mccl
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "recovery")
 }
 
-func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SDBInstanceRecoveryConfigInput) (jsonutils.JSONObject, error) {
 	if !utils.IsInStringArray(self.Status, []string{api.DBINSTANCE_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("Cannot do recovery dbinstance in status %s required status %s", self.Status, api.DBINSTANCE_RUNNING)
 	}
 
-	params := data.(*jsonutils.JSONDict)
-	backupV := validators.NewModelIdOrNameValidator("dbinstancebackup", "dbinstancebackup", userCred)
-	err := backupV.Validate(params)
+	_backup, err := DBInstanceBackupManager.FetchByIdOrName(userCred, input.DBInstancebackupId)
 	if err != nil {
-		return nil, err
-	}
-	input := &api.SDBInstanceRecoveryConfigInput{}
-	err = params.Unmarshal(input)
-	if err != nil {
-		return nil, httperrors.NewInputParameterError("Failed to unmarshal input config: %v", err)
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2("dbinstancebackup", input.DBInstancebackupId)
+		}
+		return nil, httperrors.NewGeneralError(err)
 	}
+	input.DBInstancebackupId = _backup.GetId()
 
 	databases, err := self.GetDBInstanceDatabases()
 	if err != nil {
@@ -702,7 +699,7 @@ func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.
 		dbDatabases = append(dbDatabases, database.Name)
 	}
 
-	backup := backupV.Model.(*SDBInstanceBackup)
+	backup := _backup.(*SDBInstanceBackup)
 	for src, dest := range input.Databases {
 		if len(dest) == 0 {
 			dest = src
@@ -725,6 +722,20 @@ func (self *SDBInstance) PerformRecovery(ctx context.Context, userCred mcclient.
 		return nil, httperrors.NewInputParameterError("backup and instance not in same cloudregion")
 	}
 
+	if len(backup.Engine) > 0 && backup.Engine != self.Engine {
+		return nil, httperrors.NewInputParameterError("can not recover data from diff rds engine")
+	}
+
+	driver, err := self.GetRegionDriver()
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+
+	err = driver.ValidateDBInstanceRecovery(ctx, userCred, self, backup, input)
+	if err != nil {
+		return nil, err
+	}
+
 	return nil, self.StartDBInstanceRecoveryTask(ctx, userCred, input.JSON(input), "")
 }
 

pkg/compute/models/disks.go

@@ -495,7 +495,9 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc
 			input.Hypervisor,
 		)
 	} else {
-		diskConfig.Backend = api.STORAGE_LOCAL
+		if len(diskConfig.Backend) == 0 {
+			diskConfig.Backend = api.STORAGE_LOCAL
+		}
 		serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor)
 		if err != nil {
 			return nil, err

pkg/compute/models/dynamicschedtags.go

@@ -105,7 +105,7 @@ type SDynamicschedtag struct {
 
 	// 动态调度标间的匹配条件
 	// example: host.sys_load > 1.5 || host.mem_used_percent > 0.7 => "high_load"
-	Condition string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`
+	Condition string `width:"1024" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`
 
 	// 动态调度标签对应的调度标签
 	// SchedtagId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" update:"admin"`

pkg/compute/models/elasticcache_instances.go

@@ -584,7 +584,7 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	instance.NetworkType = extInstance.GetNetworkType()
 	if instance.NetworkType == api.LB_NETWORK_TYPE_CLASSIC {
-		vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(region)
+		vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(provider, region)
 		if err != nil {
 			return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
 		}
@@ -901,7 +901,7 @@ func (self *SElasticcache) GetCreateHuaweiElasticcacheParams(data *jsonutils.JSO
 
 	// fill security group here
 	if len(self.SecurityGroupId) > 0 {
-		sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id)
+		sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id, "")
 		if err != nil {
 			return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.SecurityGroup")
 		}

pkg/compute/models/elasticips.go

@@ -156,6 +156,8 @@ func (manager *SElasticipManager) ListItemFilter(
 				sq := networks.Query(networks.Field("id")).Join(wires, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))).
 					Filter(sqlchemy.Equals(wires.Field("zone_id"), zone.Id)).SubQuery()
 				q = q.Filter(sqlchemy.In(q.Field("network_id"), sq))
+				gns := GuestnetworkManager.Query("network_id").Equals("guest_id", guest.Id).SubQuery()
+				q = q.Filter(sqlchemy.NotIn(q.Field("network_id"), gns))
 			} else {
 				region := guest.getRegion()
 				q = q.Equals("cloudregion_id", region.Id)

pkg/compute/models/globalvpcs.go

@@ -240,3 +240,12 @@ func (globalVpc *SGlobalVpc) GetRequiredSharedDomainIds() []string {
 	}
 	return db.ISharableMergeShareRequireDomainIds(requires...)
 }
+
+func (globalVpc *SGlobalVpc) GetChangeOwnerRequiredDomainIds() []string {
+	requires := stringutils2.SSortedStrings{}
+	vpcs, _ := globalVpc.GetVpcs()
+	for i := range vpcs {
+		requires = stringutils2.Append(requires, vpcs[i].DomainId)
+	}
+	return requires
+}

pkg/compute/models/guest_actions.go

@@ -4623,6 +4623,9 @@ func (self *SGuest) PerformSnapshotAndClone(
 		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
+	} else {
+		cancelRegionUsage := &SRegionQuota{Snapshot: snapshotUsage.Snapshot}
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, cancelRegionUsage, true)
 	}
 
 	err = self.StartInstanceSnapshotAndCloneTask(

pkg/compute/models/guest_template.go

@@ -315,25 +315,19 @@ func (gt *SGuestTemplate) getMoreDetails(ctx context.Context, userCred mcclient.
 		return out, err
 	}
 	configInfo := computeapis.GuestTemplateConfigInfo{}
-	if len(input.PreferRegion) != 0 {
-		region := CloudregionManager.FetchRegionById(input.PreferRegion)
-		if region != nil {
-			input.PreferRegion = region.GetName()
-		}
-		configInfo.Region = input.PreferRegion
-
-	}
 	if len(input.PreferZone) != 0 {
 		zone := ZoneManager.FetchZoneById(input.PreferZone)
 		if zone != nil {
 			input.PreferZone = zone.GetName()
+			out.ZoneId = zone.GetId()
 		}
 		out.Zone = input.PreferZone
-		configInfo.Zone = input.PreferZone
 	}
-	configInfo.Hypervisor = gt.Hypervisor
 	out.Brand = Hypervisor2Brand(gt.Hypervisor)
 
+	// metadata
+	configInfo.Metadata = input.Metadata
+
 	// sku deal
 	if len(input.InstanceType) > 0 {
 		skuOutput := computeapis.GuestTemplateSku{}

pkg/compute/models/guests.go

@@ -535,6 +535,26 @@ func (manager *SGuestManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field
 	return q, httperrors.ErrNotFound
 }
 
+func (manager *SGuestManager) InitializeData() error {
+	guests := make([]SGuest, 0, 10)
+	q := manager.Query().Equals("hypervisor", "esxi")
+	err := db.FetchModelObjects(manager, q, &guests)
+	if err != nil {
+		return errors.Wrap(err, "db.FetchModelObjects")
+	}
+	// remove secgroup for esxi guest
+	for i := range guests {
+		if len(guests[i].SecgrpId) == 0 {
+			continue
+		}
+		db.Update(&guests[i], func() error {
+			guests[i].SecgrpId = ""
+			return nil
+		})
+	}
+	return nil
+}
+
 func (guest *SGuest) GetHypervisor() string {
 	if len(guest.Hypervisor) == 0 {
 		return api.HYPERVISOR_DEFAULT
@@ -1313,6 +1333,7 @@ func (manager *SGuestManager) validateCreateData(
 
 	maxSecgrpCount := GetDriver(hypervisor).GetMaxSecurityGroupCount()
 	if maxSecgrpCount == 0 { //esxi 不支持安全组
+		input.SecgroupId = ""
 		input.Secgroups = []string{}
 	} else if len(input.Secgroups)+1 > maxSecgrpCount {
 		return nil, httperrors.NewInputParameterError("%s shall bind up to %d security groups", hypervisor, maxSecgrpCount)
@@ -1650,11 +1671,13 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 		guest.setUserData(ctx, userCred, userData)
 	}
 	secgroups, _ := jsonutils.GetStringArray(data, "secgroups")
-	for _, secgroup := range secgroups {
-		gs := SGuestsecgroup{}
-		gs.SecgroupId = secgroup
-		gs.GuestId = guest.Id
-		GuestsecgroupManager.TableSpec().Insert(&gs)
+	for _, secgroupId := range secgroups {
+		if secgroupId != guest.SecgrpId {
+			gs := SGuestsecgroup{}
+			gs.SecgroupId = secgroupId
+			gs.GuestId = guest.Id
+			GuestsecgroupManager.TableSpec().Insert(&gs)
+		}
 	}
 }
 
@@ -2703,7 +2726,8 @@ func getCloudNicNetwork(vnic cloudprovider.ICloudNic, host *SHost, ipList []stri
 	if vnet == nil {
 		if vnic.InClassicNetwork() {
 			region := host.GetRegion()
-			vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(region)
+			cloudprovider := region.GetCloudprovider()
+			vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(cloudprovider, region)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
 			}

pkg/compute/models/hosts.go

@@ -698,7 +698,7 @@ func (self *SHost) GetHoststorages() []SHoststorage {
 	hoststorages := make([]SHoststorage, 0)
 	q := self.GetHoststoragesQuery()
 	err := db.FetchModelObjects(HoststorageManager, q, &hoststorages)
-	if err != nil {
+	if err != nil && errors.Cause(err) != sql.ErrNoRows {
 		log.Errorf("GetHoststorages error %s", err)
 		return nil
 	}
@@ -710,7 +710,9 @@ func (self *SHost) GetHoststorageOfId(storageId string) *SHoststorage {
 	hoststorage.SetModelManager(HoststorageManager, &hoststorage)
 	err := self.GetHoststoragesQuery().Equals("storage_id", storageId).First(&hoststorage)
 	if err != nil {
-		log.Errorf("GetHoststorageOfId fail %s", err)
+		if errors.Cause(err) != sql.ErrNoRows {
+			log.Errorf("GetHoststorageOfId fail %s", err)
+		}
 		return nil
 	}
 	return &hoststorage
@@ -729,7 +731,9 @@ func (self *SHost) GetHoststorageByExternalId(extId string) *SHoststorage {
 
 	err := q.First(&hoststorage)
 	if err != nil {
-		log.Errorf("GetHoststorageByExternalId fail %s", err)
+		if errors.Cause(err) != sql.ErrNoRows {
+			log.Errorf("GetHoststorageByExternalId fail %s", err)
+		}
 		return nil
 	}
 
@@ -3948,6 +3952,14 @@ func (self *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCrede
 	if wire == nil {
 		return fmt.Errorf("No wire attached")
 	}
+	if self.ZoneId == "" {
+		if _, err := self.SaveUpdates(func() error {
+			self.ZoneId = wire.ZoneId
+			return nil
+		}); err != nil {
+			return errors.Wrapf(err, "set host zone_id %s by wire", wire.ZoneId)
+		}
+	}
 	hw, err := HostwireManager.FetchByHostIdAndMac(self.Id, netif.Mac)
 	if err != nil {
 		return err
@@ -5124,6 +5136,15 @@ func (host *SHost) PerformChangeOwner(ctx context.Context, userCred mcclient.Tok
 	return ret, nil
 }
 
+func (host *SHost) GetChangeOwnerRequiredDomainIds() []string {
+	requires := stringutils2.SSortedStrings{}
+	guests := host.GetGuests()
+	for i := range guests {
+		requires = stringutils2.Append(requires, guests[i].DomainId)
+	}
+	return requires
+}
+
 func GetHostQuotaKeysFromCreateInput(input api.HostCreateInput) quotas.SDomainRegionalCloudResourceKeys {
 	ownerId := &db.SOwnerId{DomainId: input.ProjectDomain}
 	var zone *SZone

pkg/compute/models/initdb.go

@@ -44,6 +44,7 @@ func InitDB() error {
 		SecurityGroupManager,
 		SecurityGroupCacheManager,
 		NetworkManager,
+		GuestManager,
 		LoadbalancerCertificateManager,
 		LoadbalancerAclManager,
 		LoadbalancerManager,

pkg/compute/models/loadbalanceracls.go

@@ -363,22 +363,27 @@ func (lbacl *SLoadbalancerAcl) PerformPatch(ctx context.Context, userCred mcclie
 }
 
 func (lbacl *SLoadbalancerAcl) ValidateDeleteCondition(ctx context.Context) error {
-	man := CachedLoadbalancerAclManager
-	t := man.TableSpec().Instance()
-	pdF := t.Field("pending_deleted")
+	men := []db.IModelManager{
+		LoadbalancerListenerManager,
+		CachedLoadbalancerAclManager,
+	}
+
 	lbaclId := lbacl.Id
-	n, err := t.Query().
-		Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))).
-		Equals("acl_id", lbaclId).
-		CountWithError()
-	if err != nil {
-		return httperrors.NewInternalServerError("get acl count fail %s", err)
-	}
-	if n > 0 {
-		// return fmt.Errorf("acl %s is still referred to by %d %s",
-		// 	lbaclId, n, man.KeywordPlural())
-		return httperrors.NewResourceBusyError("acl %s is still referred to by %d %s", lbaclId, n, man.KeywordPlural())
+	for _, man := range men {
+		t := man.TableSpec().Instance()
+		pdF := t.Field("pending_deleted")
+		n, err := t.Query().
+			Filter(sqlchemy.OR(sqlchemy.IsNull(pdF), sqlchemy.IsFalse(pdF))).
+			Equals("acl_id", lbaclId).
+			CountWithError()
+		if err != nil {
+			return httperrors.NewInternalServerError("get acl count fail %s", err)
+		}
+		if n > 0 {
+			return httperrors.NewResourceBusyError("acl %s is still referred to by %d %s", lbaclId, n, man.KeywordPlural())
+		}
 	}
+
 	return nil
 }
 

pkg/compute/models/networks.go

@@ -704,7 +704,14 @@ func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien
 	SyncCloudProject(userCred, self, syncOwnerId, extNet, vpc.ManagerId)
 
 	if provider != nil {
-		self.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
+		shareInfo := provider.getAccountShareInfo()
+		if utils.IsInStringArray(provider.Provider, api.PRIVATE_CLOUD_PROVIDERS) && extNet.GetPublicScope() == rbacutils.ScopeNone {
+			shareInfo = apis.SAccountShareInfo{
+				IsPublic:    false,
+				PublicScope: rbacutils.ScopeNone,
+			}
+		}
+		self.SyncShareState(ctx, userCred, shareInfo)
 	}
 
 	return nil
@@ -746,7 +753,14 @@ func (manager *SNetworkManager) newFromCloudNetwork(ctx context.Context, userCre
 	SyncCloudProject(userCred, &net, syncOwnerId, extNet, vpc.ManagerId)
 
 	if provider != nil {
-		net.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
+		shareInfo := provider.getAccountShareInfo()
+		if utils.IsInStringArray(provider.Provider, api.PRIVATE_CLOUD_PROVIDERS) && extNet.GetPublicScope() == rbacutils.ScopeNone {
+			shareInfo = apis.SAccountShareInfo{
+				IsPublic:    false,
+				PublicScope: rbacutils.ScopeNone,
+			}
+		}
+		net.SyncShareState(ctx, userCred, shareInfo)
 	}
 
 	db.OpsLog.LogEvent(&net, db.ACT_CREATE, net.GetShortDesc(ctx), userCred)
@@ -1435,12 +1449,12 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 		}
 	}
 	{
-		nets := manager.getAllNetworks(wire.Id, "")
-		if nets == nil {
-			return input, httperrors.NewInternalServerError("query all networks fail")
+		nets, err := vpc.GetNetworks()
+		if err != nil {
+			return input, httperrors.NewInternalServerError("fail to GetNetworks of vpc: %v", err)
 		}
 		if isOverlapNetworks(nets, ipStart, ipEnd) {
-			return input, httperrors.NewInputParameterError("Conflict address space with existing networks")
+			return input, httperrors.NewInputParameterError("Conflict address space with existing networks in vpc %q", vpc.GetName())
 		}
 	}
 

pkg/compute/models/regiondrivers.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type IRegionDriver interface {
@@ -117,8 +118,8 @@ type IRegionDriver interface {
 	RequestUnBindIPFromNatgateway(ctx context.Context, task taskman.ITask, nat INatHelper, natgateway *SNatGateway) error
 	BindIPToNatgatewayRollback(ctx context.Context, eipId string) error
 
-	RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, task taskman.ITask) error
-	RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup) (string, error)
+	RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, removeProjectId string, task taskman.ITask) error
+	RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup, removeProjectId string) (string, error)
 	GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder // Desc(priority值越大,优先级越高) Asc(priority值越小,优先级越高)
 	GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule
 	GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule
@@ -131,6 +132,7 @@ type IRegionDriver interface {
 	IsSecurityGroupBelongGlobalVpc() bool //安全组子账号范围内可用
 	GetDefaultSecurityGroupVpcId() string
 	GetSecurityGroupVpcId(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, host *SHost, vpc *SVpc, classic bool) (string, error)
+	GetSecurityGroupPublicScope() rbacutils.TRbacScope
 
 	IsSupportedBillingCycle(bc billing.SBillingCycle, resource string) bool
 	GetSecgroupVpcid(vpcId string) string
@@ -162,6 +164,8 @@ type IDBInstanceDriver interface {
 
 	InitDBInstanceUser(dbinstance *SDBInstance, task taskman.ITask, desc *cloudprovider.SManagedDBInstanceCreateConfig) error
 	IsDBInstanceNeedSecgroup() bool
+
+	ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, backup *SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error
 }
 
 type IElasticcacheDriver interface {

pkg/compute/models/scaling_group.go

@@ -373,6 +373,21 @@ func (sgm *SScalingGroupManager) FetchCustomizeColumns(
 		n, _ = sg.ScalingPolicyNumber()
 		rows[i].ScalingPolicyNumber = n
 		rows[i].Brand = Hypervisor2Brand(sg.Hypervisor)
+		nets, err := sg.Networks()
+		if err != nil {
+			log.Errorf("sg.Networks error: %s", err)
+			continue
+		}
+		sgNets := make([]api.ScalingGroupNetwork, 0, len(nets))
+		for i := range nets {
+			sgNets = append(sgNets, api.ScalingGroupNetwork{
+				Id:           nets[i].GetId(),
+				Name:         nets[i].GetName(),
+				GuestIpStart: nets[i].GuestIpStart,
+				GuestIpEnd:   nets[i].GuestIpEnd,
+			})
+		}
+		rows[i].Networks = sgNets
 	}
 	return rows
 }
@@ -704,6 +719,18 @@ func (s *SGuest) PerformDetachScalingGroup(ctx context.Context, userCred mcclien
 	return nil, nil
 }
 
+func (sg *SScalingGroup) Networks() ([]SNetwork, error) {
+	nets := make([]SNetwork, 0, 1)
+	sgnQuery := ScalingGroupNetworkManager.Query("network_id").Equals("scaling_group_id", sg.Id).SubQuery()
+	netQuery := NetworkManager.Query().In("id", sgnQuery)
+
+	err := db.FetchModelObjects(NetworkManager, netQuery, &nets)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return nets, nil
+}
+
 func (sg *SScalingGroup) NetworkIds() ([]string, error) {
 	sgnQuery := ScalingGroupNetworkManager.Query("network_id").Equals("scaling_group_id", sg.Id)
 	rows, err := sgnQuery.Rows()

pkg/compute/models/schedpolicies.go

@@ -58,7 +58,7 @@ type SSchedpolicy struct {
 	db.SStandaloneResourceBase
 	SSchedtagResourceBase
 
-	Condition string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	Condition string `width:"1024" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
 	Strategy  string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
 
 	Enabled tristate.TriState `nullable:"false" default:"true" create:"optional" list:"user" update:"user"`

pkg/compute/models/secgroupcache.go

@@ -57,7 +57,8 @@ type SSecurityGroupCache struct {
 	// SecgroupId string `width:"128" charset:"ascii" list:"user" create:"required"`
 
 	// 虚拟私有网络外部Id
-	VpcId string `width:"128" charset:"ascii" list:"user" create:"required"`
+	VpcId             string `width:"128" charset:"ascii" list:"user" create:"required"`
+	ExternalProjectId string `width:"128" charset:"ascii" list:"user" create:"optional"`
 }
 
 var SecurityGroupCacheManager *SSecurityGroupCacheManager
@@ -260,7 +261,7 @@ func (manager *SSecurityGroupCacheManager) FetchCustomizeColumns(
 	return rows
 }
 
-func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string) (*SSecurityGroupCache, error) {
+func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) {
 	secgroupCache := SSecurityGroupCache{}
 	query := manager.Query()
 	conds := []sqlchemy.ICondition{
@@ -268,6 +269,9 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context,
 		sqlchemy.Equals(query.Field("vpc_id"), vpcId),
 		sqlchemy.Equals(query.Field("manager_id"), providerId),
 	}
+	if len(projectId) > 0 {
+		conds = append(conds, sqlchemy.Equals(query.Field("external_project_id"), projectId))
+	}
 	_region, err := CloudregionManager.FetchById(regionId)
 	if err != nil {
 		return nil, errors.Wrapf(err, "CloudregionManager.FetchById(%s)", regionId)
@@ -290,7 +294,7 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context,
 	return &secgroupCache, nil
 }
 
-func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) (*SSecurityGroupCache, error) {
+func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) {
 	lockman.LockClass(ctx, manager, userCred.GetProjectId())
 	defer lockman.ReleaseClass(ctx, manager, userCred.GetProjectId())
 
@@ -306,6 +310,7 @@ func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCre
 	secgroupCache.Status = api.SECGROUP_CACHE_STATUS_CACHING
 	secgroupCache.CloudregionId = regionId
 	secgroupCache.Name = secgroup.GetName()
+	secgroupCache.ExternalProjectId = projectId
 	secgroupCache.SetModelManager(manager, secgroupCache)
 	if err := manager.TableSpec().Insert(secgroupCache); err != nil {
 		log.Errorf("insert secgroupcache error: %v", err)
@@ -314,8 +319,8 @@ func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCre
 	return secgroupCache, nil
 }
 
-func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) (*SSecurityGroupCache, error) {
-	secgroupCache, err := manager.GetSecgroupCache(ctx, userCred, secgroupId, vpcId, regionId, providerId)
+func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) {
+	secgroupCache, err := manager.GetSecgroupCache(ctx, userCred, secgroupId, vpcId, regionId, providerId, projectId)
 	if err != nil {
 		return nil, err
 	}
@@ -324,7 +329,7 @@ func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCre
 		return secgroupCache, nil
 	}
 
-	return manager.NewCache(ctx, userCred, secgroupId, vpcId, regionId, providerId)
+	return manager.NewCache(ctx, userCred, secgroupId, vpcId, regionId, providerId, projectId)
 }
 
 func (manager *SSecurityGroupCacheManager) getSecgroupcachesByProvider(provider *SCloudprovider, region *SCloudregion, vpcId string) ([]SSecurityGroupCache, error) {
@@ -414,6 +419,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 			commondb[i].Status = api.SECGROUP_CACHE_STATUS_READY
 			commondb[i].Name = commonext[i].GetName()
 			commondb[i].Description = commonext[i].GetDescription()
+			commondb[i].ExternalProjectId = commonext[i].GetProjectId()
 			return nil
 		})
 		if err != nil {
@@ -442,7 +448,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C
 				log.Warningf("failed to set secgroup %s(%s) project sharable", secgroup.Name, secgroup.Id)
 			}
 		}
-		cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id)
+		cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id, added[i].GetProjectId())
 		if err != nil {
 			syncResult.AddError(fmt.Errorf("failed to create secgroup cache for secgroup %s(%s) provider: %s: %s", secgroup.Name, secgroup.Name, provider.Name, err))
 			continue

pkg/compute/models/secgroupresource.go

@@ -118,14 +118,6 @@ func (manager *SSecurityGroupResourceBaseManager) ListItemFilter(
 		sq := SecurityGroupManager.Query("id").Like("name", "%"+query.SecgroupName+"%")
 		q = q.In("secgroup_id", sq.SubQuery())
 	}
-	if len(query.Project) > 0 {
-		tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, query.Project)
-		if err != nil {
-			return nil, httperrors.NewResourceNotFoundError2("projects", query.Project)
-		}
-		sq := SecurityGroupManager.Query("id").Equals("tenant_id", tenant.Id)
-		q = q.In("secgroup_id", sq.SubQuery())
-	}
 	return q, nil
 }
 

pkg/compute/models/secgrouprules.go

@@ -123,12 +123,9 @@ func (manager *SSecurityGroupRuleManager) FetchOwnerId(ctx context.Context, data
 }
 
 func (manager *SSecurityGroupRuleManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
-	if userCred != nil {
-		sq := SecurityGroupManager.Query("id")
-		ssq := SecurityGroupManager.FilterByOwner(sq, userCred, scope)
-		return q.In("secgroup_id", ssq.SubQuery())
-	}
-	return q
+	sq := SecurityGroupManager.Query("id")
+	sq = db.SharableManagerFilterByOwner(SecurityGroupManager, sq, userCred, scope)
+	return q.In("secgroup_id", sq.SubQuery())
 }
 
 func (manager *SSecurityGroupRuleManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
@@ -179,6 +176,16 @@ func (manager *SSecurityGroupRuleManager) ListItemFilter(
 	if err != nil {
 		return nil, errors.Wrap(err, "SSecurityGroupResourceBaseManager.ListItemFilter")
 	}
+	if len(query.Projects) > 0 {
+		sq := SecurityGroupManager.Query("id")
+		tenants := db.TenantCacheManager.GetTenantQuery().SubQuery()
+		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
+			sqlchemy.In(tenants.Field("id"), query.Projects),
+			sqlchemy.In(tenants.Field("name"), query.Projects),
+		)).SubQuery()
+		sq = sq.In("tenant_id", subq)
+		sql = sql.In("secgroup_id", sq.SubQuery())
+	}
 	if len(query.Direction) > 0 {
 		sql = sql.Equals("direction", query.Direction)
 	}

pkg/compute/models/secgroups.go

@@ -312,6 +312,14 @@ func (manager *SSecurityGroupManager) FetchCustomizeColumns(
 		sqlchemy.In(q.Field("secgrp_id"), secgroupIds),
 		sqlchemy.In(q.Field("admin_secgrp_id"), secgroupIds),
 	))
+
+	ownerId, queryScope, err := db.FetchCheckQueryOwnerScope(ctx, userCred, query, GuestManager, policy.PolicyActionList, true)
+	if err != nil {
+		log.Errorf("FetchCheckQueryOwnerScope error: %v", err)
+		return rows
+	}
+
+	q = GuestManager.FilterByOwner(q, ownerId, queryScope)
 	err = db.FetchModelObjects(GuestManager, q, &guests)
 	if err != nil {
 		log.Errorf("db.FetchModelObjects error: %v", err)
@@ -341,8 +349,11 @@ func (manager *SSecurityGroupManager) FetchCustomizeColumns(
 		}
 	}
 
+	sq := GuestManager.Query("id")
+	sq = GuestManager.FilterByOwner(sq, ownerId, queryScope)
+
 	guestSecgroups := []SGuestsecgroup{}
-	q = GuestsecgroupManager.Query().In("secgroup_id", secgroupIds)
+	q = GuestsecgroupManager.Query().In("secgroup_id", secgroupIds).In("guest_id", sq.SubQuery())
 	err = db.FetchModelObjects(GuestsecgroupManager, q, &guestSecgroups)
 	if err != nil {
 		log.Errorf("db.FetchModelObjects error: %v", err)

pkg/compute/models/skus.go

@@ -1203,6 +1203,8 @@ func (self *SServerSku) syncWithCloudSku(ctx context.Context, userCred mcclient.
 		self.ZoneId = extSku.ZoneId
 		self.PrepaidStatus = extSku.PrepaidStatus
 		self.PostpaidStatus = extSku.PostpaidStatus
+		self.SysDiskType = extSku.SysDiskType
+		self.DataDiskTypes = extSku.DataDiskTypes
 		return nil
 	})
 	return err

pkg/compute/models/storagecaches.go

@@ -599,6 +599,7 @@ func (cache *SStoragecache) SyncCloudImages(
 
 	syncResult := compare.SyncResult{}
 
+	log.Debugln("localCachedImages started")
 	localCachedImages := cache.getCachedImages()
 	log.Debugf("localCachedImages %d", len(localCachedImages))
 

pkg/compute/models/storages.go

@@ -1483,6 +1483,15 @@ func (storage *SStorage) performChangeOwnerInternal(ctx context.Context, userCre
 	return storage.SEnabledStatusInfrasResourceBase.PerformChangeOwner(ctx, userCred, query, input)
 }
 
+func (storage *SStorage) GetChangeOwnerRequiredDomainIds() []string {
+	requires := stringutils2.SSortedStrings{}
+	disks := storage.GetDisks()
+	for i := range disks {
+		requires = stringutils2.Append(requires, disks[i].DomainId)
+	}
+	return requires
+}
+
 func (storage *SStorage) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicDomainInput) (jsonutils.JSONObject, error) {
 	// not allow to perform public for locally connected storage
 	if storage.IsLocal() {

pkg/compute/models/vpcs.go

@@ -179,8 +179,7 @@ func (manager *SVpcManager) getVpcExternalIdForClassicNetwork(regionId, cloudpro
 	return fmt.Sprintf("%s-%s", regionId, cloudproviderId)
 }
 
-func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(region *SCloudregion) (*SVpc, error) {
-	cloudprovider := region.GetCloudprovider()
+func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(cloudprovider *SCloudprovider, region *SCloudregion) (*SVpc, error) {
 	externalId := manager.getVpcExternalIdForClassicNetwork(region.Id, cloudprovider.Id)
 	_vpc, err := db.FetchByExternalIdAndManagerId(manager, externalId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
 		return q.Equals("manager_id", region.ManagerId)
@@ -215,6 +214,16 @@ func (self *SVpc) getNetworkQuery() *sqlchemy.SQuery {
 	return q
 }
 
+func (self *SVpc) GetNetworks() ([]SNetwork, error) {
+	q := self.getNetworkQuery()
+	nets := make([]SNetwork, 0, 5)
+	err := db.FetchModelObjects(NetworkManager, q, &nets)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return nets, nil
+}
+
 func (self *SVpc) GetNetworkCount() (int, error) {
 	q := self.getNetworkQuery()
 	return q.CountWithError()
@@ -372,7 +381,7 @@ func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.Toke
 		localVPCs = append(localVPCs, commondb[i])
 		remoteVPCs = append(remoteVPCs, commonext[i])
 		syncResult.Update()
-		err = commondb[i].SyncGlobalVpc(ctx, userCred, provider.GetOwnerId())
+		err = commondb[i].SyncGlobalVpc(ctx, userCred, provider.GetOwnerId(), provider)
 		if err != nil {
 			log.Errorf("%s(%s) sync global vpc error: %v", commondb[i].Name, commondb[i].Id, err)
 		}
@@ -387,7 +396,7 @@ func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.Toke
 		localVPCs = append(localVPCs, *newVpc)
 		remoteVPCs = append(remoteVPCs, added[i])
 		syncResult.Add()
-		err = newVpc.SyncGlobalVpc(ctx, userCred, provider.GetOwnerId())
+		err = newVpc.SyncGlobalVpc(ctx, userCred, provider.GetOwnerId(), provider)
 		if err != nil {
 			log.Errorf("%s(%s) sync global vpc error: %v", newVpc.Name, newVpc.Id, err)
 		}
@@ -417,10 +426,11 @@ func (self *SVpc) syncRemoveCloudVpc(ctx context.Context, userCred mcclient.Toke
 	return err
 }
 
-func (self *SVpc) SyncGlobalVpc(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider) error {
+func (self *SVpc) SyncGlobalVpc(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, provider *SCloudprovider) error {
 	if len(self.GlobalvpcId) > 0 {
 		gv, _ := self.GetGlobalVpc()
 		SyncCloudDomain(userCred, gv, ownerId)
+		gv.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
 		return nil
 	}
 	region, err := self.GetRegion()
@@ -464,6 +474,7 @@ func (self *SVpc) SyncGlobalVpc(ctx context.Context, userCred mcclient.TokenCred
 				return errors.Wrap(err, "GlobalVpcManager.Insert")
 			}
 			SyncCloudDomain(userCred, gv, ownerId)
+			gv.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
 			globalvpcId = gv.Id
 		}
 		_, err = db.Update(self, func() error {
@@ -1096,6 +1107,15 @@ func (vpc *SVpc) GetChangeOwnerCandidateDomainIds() []string {
 	return db.ISharableMergeChangeOwnerCandidateDomainIds(vpc, candidates...)
 }
 
+func (vpc *SVpc) GetChangeOwnerRequiredDomainIds() []string {
+	requires := stringutils2.SSortedStrings{}
+	wires := vpc.GetWires()
+	for i := range wires {
+		requires = stringutils2.Append(requires, wires[i].DomainId)
+	}
+	return requires
+}
+
 func (vpc *SVpc) GetRequiredSharedDomainIds() []string {
 	wires := vpc.GetWires()
 	if len(wires) == 0 {

pkg/compute/models/wireresource.go

@@ -174,26 +174,25 @@ func (manager *SWireResourceBaseManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SVpcResourceBaseManager.ListItemFilter")
 	}
 
-	if len(query.Zone) > 0 || len(query.Zones) > 0 {
+	if len(query.ZoneList()) > 0 {
 		region := &SCloudregion{}
-		if len(query.Zone) > 0 {
-			sq := ZoneManager.Query().SubQuery()
-			q := CloudregionManager.Query()
-			q = q.Join(sq, sqlchemy.Equals(sq.Field("cloudregion_id"), q.Field("id"))).Filter(sqlchemy.OR(
-				sqlchemy.Equals(sq.Field("id"), query.Zone),
-				sqlchemy.Equals(sq.Field("name"), query.Zone),
-			))
-			count, err := q.CountWithError()
-			if err != nil {
-				return nil, errors.Wrap(err, "CountWithError")
-			}
-			if count < 1 {
-				return nil, httperrors.NewResourceNotFoundError2("zone", query.Zone)
-			}
-			err = q.First(region)
-			if err != nil {
-				return nil, errors.Wrap(err, "q.First")
-			}
+		firstZone := query.FirstZone()
+		sq := ZoneManager.Query().SubQuery()
+		q := CloudregionManager.Query()
+		q = q.Join(sq, sqlchemy.Equals(sq.Field("cloudregion_id"), q.Field("id"))).Filter(sqlchemy.OR(
+			sqlchemy.Equals(sq.Field("id"), firstZone),
+			sqlchemy.Equals(sq.Field("name"), firstZone),
+		))
+		count, err := q.CountWithError()
+		if err != nil {
+			return nil, errors.Wrap(err, "CountWithError")
+		}
+		if count < 1 {
+			return nil, httperrors.NewResourceNotFoundError2("zone", firstZone)
+		}
+		err = q.First(region)
+		if err != nil {
+			return nil, errors.Wrap(err, "q.First")
 		}
 		if utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
 			vpcQ := VpcManager.Query().SubQuery()

pkg/compute/models/wires.go

@@ -309,7 +309,7 @@ func (manager *SWireManager) SyncWires(ctx context.Context, userCred mcclient.To
 		}
 	}
 	for i := 0; i < len(commondb); i += 1 {
-		err = commondb[i].syncWithCloudWire(ctx, userCred, commonext[i], provider)
+		err = commondb[i].syncWithCloudWire(ctx, userCred, commonext[i], vpc, provider)
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
@@ -353,13 +353,19 @@ func (self *SWire) syncRemoveCloudWire(ctx context.Context, userCred mcclient.To
 	return err
 }
 
-func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.TokenCredential, extWire cloudprovider.ICloudWire, provider *SCloudprovider) error {
+func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.TokenCredential, extWire cloudprovider.ICloudWire, vpc *SVpc, provider *SCloudprovider) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		// self.Name = extWire.GetName()
 		self.Bandwidth = extWire.GetBandwidth() // 10G
 
 		self.IsEmulated = extWire.IsEmulated()
 
+		if self.IsEmulated {
+			self.DomainId = vpc.DomainId
+			self.IsPublic = vpc.IsPublic
+			self.PublicScope = vpc.PublicScope
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -409,13 +415,17 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl
 
 	wire.IsEmulated = extWire.IsEmulated()
 
+	wire.DomainId = vpc.DomainId
+	wire.IsPublic = vpc.IsPublic
+	wire.PublicScope = vpc.PublicScope
+
 	err = manager.TableSpec().Insert(&wire)
 	if err != nil {
 		log.Errorf("newFromCloudWire fail %s", err)
 		return nil, err
 	}
 
-	if provider != nil {
+	if provider != nil && !wire.IsEmulated {
 		SyncCloudDomain(userCred, &wire, provider.GetOwnerId())
 		wire.SyncShareState(ctx, userCred, provider.getAccountShareInfo())
 	}
@@ -1024,6 +1034,15 @@ func (wire *SWire) GetChangeOwnerCandidateDomainIds() []string {
 	return db.ISharableMergeChangeOwnerCandidateDomainIds(wire, candidates...)
 }
 
+func (wire *SWire) GetChangeOwnerRequiredDomainIds() []string {
+	requires := stringutils2.SSortedStrings{}
+	networks, _ := wire.getNetworks(nil, rbacutils.ScopeNone)
+	for i := range networks {
+		requires = stringutils2.Append(requires, networks[i].DomainId)
+	}
+	return requires
+}
+
 func (wire *SWire) GetRequiredSharedDomainIds() []string {
 	networks, _ := wire.getNetworks(nil, rbacutils.ScopeNone)
 	if len(networks) == 0 {

pkg/compute/policy/defaults.go

@@ -47,12 +47,6 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
-				{
-					Service:  api.SERVICE_TYPE,
-					Resource: "metadatas",
-					Action:   PolicyActionList,
-					Result:   rbacutils.Allow,
-				},
 				{
 					Service:  api.SERVICE_TYPE,
 					Resource: "cloudregions",

pkg/compute/policy/resources.go

@@ -34,6 +34,7 @@ var (
 		"loadbalanceragents",
 		// "reservedips",
 		"policy_definitions",
+		"schedtags",
 	}
 	computeDomainResources = []string{
 		"cloudaccounts",

pkg/compute/regiondrivers/aliyun.go

@@ -975,6 +975,26 @@ func (self *SAliyunRegionDriver) IsSecurityGroupBelongVpc() bool {
 	return true
 }
 
+func (self *SAliyunRegionDriver) ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, backup *models.SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error {
+	if !utils.IsInStringArray(instance.Engine, []string{api.DBINSTANCE_TYPE_MYSQL, api.DBINSTANCE_TYPE_SQLSERVER}) {
+		return httperrors.NewNotSupportedError("Aliyun %s not support recovery", instance.Engine)
+	}
+	if instance.Engine == api.DBINSTANCE_TYPE_MYSQL {
+		if backup.DBInstanceId != instance.Id {
+			return httperrors.NewUnsupportOperationError("Aliyun %s only support recover from it self backups", instance.Engine)
+		}
+		if !((utils.IsInStringArray(instance.EngineVersion, []string{"8.0", "5.7"}) &&
+			instance.StorageType == api.ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD &&
+			instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA) || (instance.EngineVersion == "5.6" && instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA)) {
+			return httperrors.NewUnsupportOperationError("Aliyun %s only 8.0 and 5.7 high_availability local_ssd or 5.6 high_availability support recovery from it self backups", instance.Engine)
+		}
+	}
+	if len(input.Databases) == 0 {
+		return httperrors.NewMissingParameterError("databases")
+	}
+	return nil
+}
+
 func (self *SAliyunRegionDriver) ValidateCreateDBInstanceData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.DBInstanceCreateInput, skus []models.SDBInstanceSku, network *models.SNetwork) (api.DBInstanceCreateInput, error) {
 	if input.BillingType == billing_api.BILLING_TYPE_PREPAID && len(input.MasterInstanceId) > 0 {
 		return input, httperrors.NewInputParameterError("slave dbinstance not support prepaid billing type")
@@ -1105,6 +1125,7 @@ func (self *SAliyunRegionDriver) RequestCreateDBInstanceBackup(ctx context.Conte
 		}
 		result := models.DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, backup.GetCloudprovider(), instance, backup.GetRegion(), backups)
 		log.Infof("SyncDBInstanceBackups for dbinstance %s(%s) result: %s", instance.Name, instance.Id, result.Result())
+		instance.SetStatus(userCred, api.DBINSTANCE_RUNNING, "")
 		return nil, nil
 	})
 	return nil

pkg/compute/regiondrivers/aws.go

@@ -568,12 +568,11 @@ func (self *SAwsRegionDriver) validateUpdateNetworkListenerData(ctx context.Cont
 
 func (self *SAwsRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, lblis *models.SLoadbalancerListener, backendGroup db.IModel) (*jsonutils.JSONDict, error) {
 	ownerId := lblis.GetOwnerId()
-	lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerId)
-	if err := lbV.Validate(data); err != nil {
-		return nil, err
+	lb := lblis.GetLoadbalancer()
+	if lb == nil {
+		return nil, httperrors.NewResourceNotFoundError("loadbalancer listener %s related listener %s not found", lblis.Id, lblis.LoadbalancerId)
 	}
 
-	lb := lbV.Model.(*models.SLoadbalancer)
 	if lb.LoadbalancerSpec == api.LB_AWS_SPEC_APPLICATION {
 		if _, err := self.validateUpdateApplicationListenerData(ctx, ownerId, lb, lblis, backendGroup, data); err != nil {
 			return nil, err
@@ -940,7 +939,7 @@ func (self *SAwsRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Conte
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil, lb.GetCloudprovider()); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId(), lb.GetCloudprovider()); err != nil {
 				return nil, errors.Wrap(err, "AwsRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -1153,7 +1152,7 @@ func (self *SAwsRegionDriver) RequestCreateLoadbalancerListenerRule(ctx context.
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil, loadbalancer.GetCloudprovider())
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }
@@ -1381,7 +1380,7 @@ func (self *SAwsRegionDriver) RequestSyncLoadbalancerListener(ctx context.Contex
 		if err := iListener.Refresh(); err != nil {
 			return nil, errors.Wrap(err, "awsRegionDriver.RequestSyncLoadbalancerListener.Refresh")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil, loadbalancer.GetCloudprovider())
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }

pkg/compute/regiondrivers/base.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -29,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SBaseRegionDriver struct {
@@ -216,7 +218,7 @@ func (self *SBaseRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mc
 	return fmt.Errorf("Not implement RequestDeleteVpc")
 }
 
-func (self *SBaseRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, task taskman.ITask) error {
+func (self *SBaseRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, remoteProjectId string, task taskman.ITask) error {
 	return fmt.Errorf("Not Implemented RequestCacheSecurityGroup")
 }
 
@@ -240,11 +242,15 @@ func (self *SBaseRegionDriver) GetDefaultSecurityGroupVpcId() string {
 	return api.NORMAL_VPC_ID
 }
 
+func (self *SBaseRegionDriver) GetSecurityGroupPublicScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeSystem
+}
+
 func (self *SBaseRegionDriver) GetSecurityGroupVpcId(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, host *models.SHost, vpc *models.SVpc, classic bool) (string, error) {
 	return "", cloudprovider.ErrNotImplemented
 }
 
-func (self *SBaseRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup) (string, error) {
+func (self *SBaseRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup, removeProjectId string) (string, error) {
 	return "", fmt.Errorf("Not Implemented RequestSyncSecurityGroup")
 }
 
@@ -336,6 +342,10 @@ func (self *SBaseRegionDriver) IsSupportKeepDBInstanceManualBackup() bool {
 	return false
 }
 
+func (self *SBaseRegionDriver) ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, backup *models.SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error {
+	return errors.Wrap(cloudprovider.ErrNotImplemented, "ValidateDBInstanceRecovery")
+}
+
 func (self *SBaseRegionDriver) IsSupportedDBInstance() bool {
 	return false
 }

pkg/compute/regiondrivers/google.go

@@ -184,6 +184,10 @@ func (self *SGoogleRegionDriver) IsSupportedDBInstance() bool {
 	return true
 }
 
+func (self *SGoogleRegionDriver) ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, backup *models.SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error {
+	return nil
+}
+
 func (self *SGoogleRegionDriver) ValidateCreateDBInstanceData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.DBInstanceCreateInput, skus []models.SDBInstanceSku, network *models.SNetwork) (api.DBInstanceCreateInput, error) {
 	if input.BillingType == billing_api.BILLING_TYPE_PREPAID {
 		return input, httperrors.NewInputParameterError("Google dbinstance not support prepaid billing type")
@@ -261,6 +265,7 @@ func (self *SGoogleRegionDriver) RequestCreateDBInstanceBackup(ctx context.Conte
 
 		result := models.DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, backup.GetCloudprovider(), instance, backup.GetRegion(), backups)
 		log.Infof("SyncDBInstanceBackups for dbinstance %s(%s) result: %s", instance.Name, instance.Id, result.Result())
+		instance.SetStatus(userCred, api.DBINSTANCE_RUNNING, "")
 		return nil, nil
 	})
 	return nil

pkg/compute/regiondrivers/huawei.go

@@ -2007,7 +2007,7 @@ func (self *SHuaWeiRegionDriver) RequestSyncLoadbalancerBackend(ctx context.Cont
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestSyncLoadbalancerBackend.GetILoadbalancerBackendById")
 			}
 
-			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil)
+			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbb.GetOwnerId())
 			if err != nil {
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestSyncLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
@@ -2066,7 +2066,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Co
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil, lb.GetCloudprovider()); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId(), lb.GetCloudprovider()); err != nil {
 				return nil, errors.Wrap(err, "huaweiRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -2129,7 +2129,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancerListenerRule(ctx conte
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, errors.Wrap(err, "huaweiRegionDriver.RequestCreateLoadbalancerListenerRule.SetExternalId")
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil, loadbalancer.GetCloudprovider())
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }
@@ -2275,6 +2275,28 @@ func (self *SHuaWeiRegionDriver) ValidateDBInstanceAccountPrivilege(ctx context.
 	return nil
 }
 
+// https://support.huaweicloud.com/api-rds/rds_09_0009.html
+func (self *SHuaWeiRegionDriver) ValidateDBInstanceRecovery(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, backup *models.SDBInstanceBackup, input api.SDBInstanceRecoveryConfigInput) error {
+	if backup.Engine == api.DBINSTANCE_TYPE_POSTGRESQL {
+		return httperrors.NewNotSupportedError("%s not support recovery", backup.Engine)
+	}
+	if backup.DBInstanceId == instance.Id && instance.Engine != api.DBINSTANCE_TYPE_SQLSERVER {
+		return httperrors.NewNotSupportedError("Huawei %s rds not support recovery from it self rds backup", instance.Engine)
+	}
+	if len(input.Databases) > 0 {
+		if instance.Engine != api.DBINSTANCE_TYPE_SQLSERVER {
+			return httperrors.NewInputParameterError("Huawei only %s engine support databases recovery", instance.Engine)
+		}
+		invalidDbs := []string{"rdsadmin", "master", "msdb", "tempdb", "model"}
+		for _, db := range input.Databases {
+			if utils.IsInStringArray(strings.ToLower(db), invalidDbs) {
+				return httperrors.NewInputParameterError("New databases name can not be one of %s", invalidDbs)
+			}
+		}
+	}
+	return nil
+}
+
 func validatorSlaveZones(ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict, optional bool) error {
 	s, err := data.GetString("slave_zones")
 	if err != nil {

pkg/compute/regiondrivers/managedvirtual.go

@@ -39,6 +39,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/rand"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SManagedVirtualizationRegionDriver struct {
@@ -597,7 +598,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerBackend
 		if err := db.SetExternalId(lbb, userCred, iLoadbalancerBackend.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, nil, lb.GetCloudprovider())
+		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iLoadbalancerBackend, lbbg.GetOwnerId(), lb.GetCloudprovider())
 	})
 	return nil
 }
@@ -682,7 +683,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncLoadbalancerBackend(c
 			return nil, errors.Wrap(err, "regionDriver.RequestSyncLoadbalancerBackend.GetILoadbalancerBackendById")
 		}
 
-		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil, lb.GetCloudprovider())
+		return nil, lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbbg.GetOwnerId(), lb.GetCloudprovider())
 	})
 	return nil
 }
@@ -782,7 +783,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene
 		if err := db.SetExternalId(lblis, userCred, iListener.GetGlobalId()); err != nil {
 			return nil, errors.Wrap(err, "db.SetExternalId")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil, lblis.GetCloudprovider())
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId(), lblis.GetCloudprovider())
 	})
 	return nil
 }
@@ -953,7 +954,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncLoadbalancerListener(
 		if err := iListener.Refresh(); err != nil {
 			return nil, errors.Wrap(err, "regionDriver.RequestSyncLoadbalancerListener.RefreshListener")
 		}
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil, lblis.GetCloudprovider())
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId(), lblis.GetCloudprovider())
 	})
 	return nil
 }
@@ -1050,7 +1051,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancerListene
 		if err := db.SetExternalId(lbr, userCred, iListenerRule.GetGlobalId()); err != nil {
 			return nil, err
 		}
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil, loadbalancer.GetCloudprovider())
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }
@@ -1457,7 +1458,7 @@ func (self *SManagedVirtualizationRegionDriver) GetSecurityGroupVpcId(ctx contex
 	return region.GetDriver().GetDefaultSecurityGroupVpcId(), nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup) (string, error) {
+func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup, remoteProjectId string) (string, error) {
 	lockman.LockRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s-%s", secgroup.Id, vpcId, vpc.ManagerId))
 	defer lockman.ReleaseRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s-%s", secgroup.Id, vpcId, vpc.ManagerId))
 
@@ -1466,7 +1467,11 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 		return "", errors.Wrap(err, "vpc.GetRegon")
 	}
 
-	cache, err := models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId)
+	if region.GetDriver().GetSecurityGroupPublicScope() == rbacutils.ScopeSystem {
+		remoteProjectId = ""
+	}
+
+	cache, err := models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId, remoteProjectId)
 	if err != nil {
 		return "", errors.Wrap(err, "SSecurityGroupCache.Register")
 	}
@@ -1480,7 +1485,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 	if len(cache.ExternalId) > 0 {
 		iSecgroup, err = iRegion.GetISecurityGroupById(cache.ExternalId)
 		if err != nil {
-			if err != cloudprovider.ErrNotFound {
+			if errors.Cause(err) != cloudprovider.ErrNotFound {
 				return "", errors.Wrap(err, "iRegion.GetSecurityGroupById")
 			}
 			cache.ExternalId = ""
@@ -1495,9 +1500,13 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 		randomString := func(prefix string, length int) string {
 			return fmt.Sprintf("%s-%s", prefix, rand.String(length))
 		}
-		groupName := randomString(secgroup.Name, 1)
+		opts := &cloudprovider.SecurityGroupFilterOptions{
+			Name:      randomString(secgroup.Name, 1),
+			VpcId:     vpcId,
+			ProjectId: remoteProjectId,
+		}
 		for i := 2; i < 30; i++ {
-			_, err := iRegion.GetISecurityGroupByName(vpc.ExternalId, groupName)
+			_, err := iRegion.GetISecurityGroupByName(opts)
 			if err != nil {
 				if errors.Cause(err) == cloudprovider.ErrNotFound {
 					break
@@ -1506,13 +1515,14 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 					return "", err
 				}
 			}
-			groupName = randomString(secgroup.Name, i)
+			opts.Name = randomString(secgroup.Name, i)
 		}
 		conf := &cloudprovider.SecurityGroupCreateInput{
-			Name:  groupName,
-			Desc:  secgroup.Description,
-			VpcId: vpcId,
-			Rules: secgroup.GetSecRules(""),
+			Name:      opts.Name,
+			Desc:      secgroup.Description,
+			VpcId:     vpcId,
+			ProjectId: remoteProjectId,
+			Rules:     secgroup.GetSecRules(""),
 		}
 		iSecgroup, err = iRegion.CreateISecurityGroup(conf)
 		if err != nil {
@@ -1560,14 +1570,13 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con
 	return cache.ExternalId, nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, task taskman.ITask) error {
-
+func (self *SManagedVirtualizationRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, removeProjectId string, task taskman.ITask) error {
 	vpcId, err := region.GetDriver().GetSecurityGroupVpcId(ctx, userCred, region, nil, vpc, classic)
 	if err != nil {
 		return errors.Wrap(err, "GetSecurityGroupVpcId")
 	}
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-		_, err := self.RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup)
+		_, err := self.RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup, removeProjectId)
 		return nil, err
 	})
 	return nil
@@ -1636,7 +1645,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstance(ctx cont
 				if err != nil {
 					return nil, errors.Wrap(err, "GetSecurityGroupVpcId")
 				}
-				desc.SecgroupId, err = region.GetDriver().RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup)
+				desc.SecgroupId, err = region.GetDriver().RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup, desc.ProjectId)
 				if err != nil {
 					return nil, errors.Wrap(err, "SyncSecurityGroup")
 				}
@@ -2326,8 +2335,12 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstanceBackup(ct
 			backup.BackupSizeMb = iBackup.GetBackupSizeMb()
 			return nil
 		})
+		if err != nil {
+			return nil, errors.Wrap(err, "db.Update")
+		}
 
-		return nil, err
+		instance.SetStatus(userCred, api.DBINSTANCE_RUNNING, "")
+		return nil, nil
 	})
 	return nil
 }

pkg/compute/regiondrivers/openstack.go

@@ -26,6 +26,7 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SOpenStackRegionDriver struct {
@@ -61,6 +62,10 @@ func (self *SOpenStackRegionDriver) IsOnlySupportAllowRules() bool {
 	return true
 }
 
+func (self *SOpenStackRegionDriver) GetSecurityGroupPublicScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeProject
+}
+
 func (self *SOpenStackRegionDriver) GetProvider() string {
 	return api.CLOUD_PROVIDER_OPENSTACK
 }

pkg/compute/regiondrivers/qcloud.go

@@ -323,7 +323,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Co
 		}
 
 		if ibackend != nil {
-			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, nil, lb.GetCloudprovider()); err != nil {
+			if err := lbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, ibackend, lbbg.GetOwnerId(), lb.GetCloudprovider()); err != nil {
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestCreateLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
 		}
@@ -610,7 +610,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListener(ctx context.C
 			}
 		}
 
-		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil, loadbalancer.GetCloudprovider())
+		return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }
@@ -704,7 +704,7 @@ func (self *SQcloudRegionDriver) RequestCreateLoadbalancerListenerRule(ctx conte
 			return nil, errors.Wrap(err, "SQcloudRegionDriver.RequestCreateLoadbalancerListener.createLoadbalancerBackendGroup")
 		}
 
-		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, nil, loadbalancer.GetCloudprovider())
+		return nil, lbr.SyncWithCloudLoadbalancerListenerRule(ctx, userCred, iListenerRule, listener.GetOwnerId(), loadbalancer.GetCloudprovider())
 	})
 	return nil
 }
@@ -1124,7 +1124,7 @@ func (self *SQcloudRegionDriver) RequestSyncLoadbalancerBackend(ctx context.Cont
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestSyncLoadbalancerBackend.Refresh")
 			}
 
-			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, nil)
+			err = cachedlbb.SyncWithCloudLoadbalancerBackend(ctx, userCred, iBackend, lbb.GetOwnerId())
 			if err != nil {
 				return nil, errors.Wrap(err, "qcloudRegionDriver.RequestSyncLoadbalancerBackend.SyncWithCloudLoadbalancerBackend")
 			}
@@ -1350,7 +1350,7 @@ func (self *SQcloudRegionDriver) RequestSyncLoadbalancerListener(ctx context.Con
 		}
 
 		if utils.IsInStringArray(lblis.ListenerType, []string{api.LB_LISTENER_TYPE_UDP, api.LB_LISTENER_TYPE_TCP}) {
-			return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, nil, loadbalancer.GetCloudprovider())
+			return nil, lblis.SyncWithCloudLoadbalancerListener(ctx, userCred, loadbalancer, iListener, loadbalancer.GetOwnerId(), loadbalancer.GetCloudprovider())
 		} else {
 			// http&https listener 变更不会同步到监听规则
 			return nil, nil

pkg/compute/service/influxdb.go

@@ -22,7 +22,6 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/compute/options"
-	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 )
@@ -30,7 +29,7 @@ import (
 func setInfluxdbRetentionPolicy() error {
 	setF := func() error {
 		urls, err := auth.GetServiceURLs("influxdb", options.Options.Region, "", "")
-		if err != nil && errors.Cause(err) == httperrors.ErrNotFound {
+		if err != nil {
 			auth.ReAuth()
 		}
 		if err != nil {

pkg/compute/tasks/cloud_account_sync_task.go

@@ -44,19 +44,32 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda
 	db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNCING_HOST, "", self.UserCred)
 	// cloudaccount.MarkSyncing(self.UserCred)
 
-	// do sync
-	err := cloudaccount.SyncCallSyncAccountTask(ctx, self.UserCred)
+	self.SetStage("OnCloudaccountSyncReady", nil)
 
-	if err != nil {
-		if errors.Cause(err) != httperrors.ErrConflict {
-			log.Debugf("no other sync task, mark end sync for all cloudproviders")
-			cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred)
+	taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) {
+		// do sync
+		err := cloudaccount.SyncCallSyncAccountTask(ctx, self.UserCred)
+
+		if err != nil {
+			if errors.Cause(err) != httperrors.ErrConflict {
+				log.Debugf("no other sync task, mark end sync for all cloudproviders")
+				cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred)
+			}
+			return nil, errors.Wrap(err, "SyncCallSyncAccountTask")
 		}
-		db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred)
-		self.SetStageFailed(ctx, err.Error())
-		logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false)
-		return
-	}
+		return nil, nil
+	})
+}
+
+func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncReadyFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	cloudaccount := obj.(*models.SCloudaccount)
+	db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred)
+	self.SetStageFailed(ctx, err.String())
+	logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false)
+}
+
+func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncReady(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	cloudaccount := obj.(*models.SCloudaccount)
 
 	driver, err := cloudaccount.GetProvider()
 	if err != nil {
@@ -111,3 +124,11 @@ func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Con
 	self.SetStageComplete(ctx, nil)
 	logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, "", self.UserCred, true)
 }
+
+func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
+	cloudaccount := obj.(*models.SCloudaccount)
+	cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred)
+	db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred)
+	self.SetStageFailed(ctx, err.String())
+	logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false)
+}

pkg/compute/tasks/dbinstance_backup_create_task.go

@@ -40,6 +40,10 @@ func (self *DBInstanceBackupCreateTask) taskFailed(ctx context.Context, backup *
 	backup.SetStatus(self.UserCred, api.DBINSTANCE_BACKUP_CREATE_FAILED, err.Error())
 	db.OpsLog.LogEvent(backup, db.ACT_CREATE, err.Error(), self.GetUserCred())
 	logclient.AddActionLogWithStartable(self, backup, logclient.ACT_CREATE, err.Error(), self.UserCred, false)
+	instance, _ := backup.GetDBInstance()
+	if instance != nil {
+		instance.SetStatus(self.UserCred, api.DBINSTANCE_BACKING_UP_FAILED, err.Error())
+	}
 	self.SetStageFailed(ctx, err.Error())
 }
 
@@ -56,7 +60,11 @@ func (self *DBInstanceBackupCreateTask) CreateDBInstanceBackup(ctx context.Conte
 	}
 
 	self.SetStage("OnCreateDBInstanceBackupComplete", nil)
-	instance.GetRegion().GetDriver().RequestCreateDBInstanceBackup(ctx, self.UserCred, instance, backup, self)
+	err = instance.GetRegion().GetDriver().RequestCreateDBInstanceBackup(ctx, self.UserCred, instance, backup, self)
+	if err != nil {
+		self.taskFailed(ctx, backup, errors.Wrap(err, "RequestCreateDBInstanceBackup"))
+		return
+	}
 }
 
 func (self *DBInstanceBackupCreateTask) OnCreateDBInstanceBackupComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/tasks/dbinstance_recovery_task.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -60,16 +61,22 @@ func (self *DBInstanceRecoveryTask) OnInit(ctx context.Context, obj db.IStandalo
 		return
 	}
 
-	backup, err := instance.GetDBInstanceBackup(input.DBInstancebackupId)
+	_backup, err := models.DBInstanceBackupManager.FetchById(input.DBInstancebackupId)
 	if err != nil {
 		self.taskFailed(ctx, instance, errors.Wrapf(err, "instance.GetDBInstanceBackup(%s)", input.DBInstancebackupId))
 		return
 	}
 
+	backup := _backup.(*models.SDBInstanceBackup)
+	origin, _ := backup.GetDBInstance()
+
 	conf := &cloudprovider.SDBInstanceRecoveryConfig{
 		BackupId:  backup.ExternalId,
 		Databases: input.Databases,
 	}
+	if origin != nil {
+		conf.OriginDBInstanceExternalId = origin.ExternalId
+	}
 
 	err = iRds.RecoveryFromBackup(conf)
 	if err != nil {
@@ -77,12 +84,21 @@ func (self *DBInstanceRecoveryTask) OnInit(ctx context.Context, obj db.IStandalo
 		return
 	}
 
-	err = cloudprovider.WaitStatus(iRds, api.DBINSTANCE_RUNNING, time.Second*10, time.Second*40)
+	err = cloudprovider.WaitStatus(iRds, api.DBINSTANCE_RUNNING, time.Second*10, time.Minute*40)
 	if err != nil {
 		self.taskFailed(ctx, instance, errors.Wrap(err, "cloudprovider.WaitStatus(running)"))
 		return
 	}
 
+	databases, err := iRds.GetIDBInstanceDatabases()
+	if err != nil {
+		log.Errorf("failed to get dbinstance %s database error: %v", instance.Name, err)
+	} else {
+		models.DBInstanceDatabaseManager.SyncDBInstanceDatabases(ctx, self.UserCred, instance, databases)
+	}
+
+	db.OpsLog.LogEvent(instance, db.ACT_RESTORE, nil, self.GetUserCred())
+	logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RESTORE, backup, self.UserCred, true)
 	instance.SetStatus(self.UserCred, api.DBINSTANCE_RUNNING, "")
 	self.SetStageComplete(ctx, nil)
 }

pkg/compute/tasks/disk_resize_task.go

@@ -23,6 +23,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -105,6 +106,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode
 		self.OnStartResizeDiskFailed(ctx, disk, err)
 		return
 	}
+	diff := int(sizeMb) - disk.DiskSize
 	oldStatus := disk.Status
 	_, err = db.Update(disk, func() error {
 		disk.Status = api.DISK_READY
@@ -119,6 +121,21 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode
 	self.SetDiskReady(ctx, disk, self.GetUserCred(), "")
 	notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status)
 	db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred)
+
+	if diff > 0 {
+		var addUsage models.SQuota
+		keys, err := disk.GetQuotaKeys()
+		if err != nil { // just log and ignore
+			log.Errorf("disk.GetQuotaKeys fail %s", err)
+		} else {
+			addUsage.SetKeys(keys)
+			addUsage.Storage = diff
+			quotas.AddUsages(ctx, self.UserCred, []db.IUsage{&addUsage})
+		}
+	} else if diff < 0 {
+		// unlikely
+	}
+
 	self.CleanHostSchedCache(disk)
 	db.OpsLog.LogEvent(disk, db.ACT_RESIZE, disk.GetShortDesc(ctx), self.UserCred)
 	logclient.AddActionLogWithStartable(self, disk, logclient.ACT_RESIZE, nil, self.UserCred, true)

pkg/compute/tasks/elasticcache_account_reset_password_task.go

@@ -41,6 +41,7 @@ func (self *ElasticcacheAccountResetPasswordTask) taskFail(ctx context.Context,
 	ec, err := db.FetchById(models.ElasticcacheManager, ea.ElasticcacheId)
 	if err == nil {
 		ec.(*models.SElasticcache).SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_CHANGE_FAILED, reason)
+		logclient.AddActionLogWithStartable(self, ec, logclient.ACT_RESET_PASSWORD, reason, self.UserCred, false)
 	}
 	db.OpsLog.LogEvent(ea, db.ACT_RESET_PASSWORD, reason, self.UserCred)
 	logclient.AddActionLogWithStartable(self, ea, logclient.ACT_RESET_PASSWORD, reason, self.UserCred, false)
@@ -65,7 +66,9 @@ func (self *ElasticcacheAccountResetPasswordTask) OnInit(ctx context.Context, ob
 		ec, err := db.FetchById(models.ElasticcacheManager, ea.ElasticcacheId)
 		if err == nil {
 			ec.(*models.SElasticcache).SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_RUNNING, "")
+			logclient.AddActionLogWithStartable(self, ec, logclient.ACT_RESET_PASSWORD, "", self.UserCred, true)
 		}
+		logclient.AddActionLogWithStartable(self, ea, logclient.ACT_RESET_PASSWORD, "", self.UserCred, true)
 		self.SetStageComplete(ctx, nil)
 	}
 }

pkg/compute/tasks/guest_change_config_task.go

@@ -199,10 +199,10 @@ func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecComplete(ctx context.C
 		return
 	}
 	changeConfigSpec := jsonutils.NewDict()
-	if addCpu != 0 {
+	if vcpuCount > 0 && addCpu != 0 {
 		changeConfigSpec.Set("add_cpu", jsonutils.NewInt(int64(addCpu)))
 	}
-	if addMem != 0 {
+	if vmemSize > 0 && addMem != 0 {
 		changeConfigSpec.Set("add_mem", jsonutils.NewInt(int64(addMem)))
 	}
 	if len(instanceType) > 0 {

pkg/compute/tasks/instance_snapshot_and_clone_task.go

@@ -50,7 +50,7 @@ func (self *InstanceSnapshotAndCloneTask) taskFailed(
 
 func (self *InstanceSnapshotAndCloneTask) taskComplete(
 	ctx context.Context, isp *models.SInstanceSnapshot, data jsonutils.JSONObject) {
-	self.finalReleasePendingUsage(ctx)
+	self.finalReleasePendingUsage(ctx, true)
 	guest := models.GuestManager.FetchGuestById(isp.GuestId)
 	guest.StartSyncstatus(ctx, self.UserCred, "")
 	db.OpsLog.LogEvent(guest, db.ACT_VM_SNAPSHOT_AND_CLONE, "", self.UserCred)
@@ -60,21 +60,21 @@ func (self *InstanceSnapshotAndCloneTask) taskComplete(
 }
 
 func (self *InstanceSnapshotAndCloneTask) SetStageFailed(ctx context.Context, reason string) {
-	self.finalReleasePendingUsage(ctx)
+	self.finalReleasePendingUsage(ctx, false)
 	self.STask.SetStageFailed(ctx, reason)
 }
 
-func (self *InstanceSnapshotAndCloneTask) finalReleasePendingUsage(ctx context.Context) {
+func (self *InstanceSnapshotAndCloneTask) finalReleasePendingUsage(ctx context.Context, success bool) {
 	pendingUsage := models.SQuota{}
 	err := self.GetPendingUsage(&pendingUsage, 0)
 	if err == nil && !pendingUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // failure cleanup
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, success)
 	}
 
 	pendingRegionUsage := models.SRegionQuota{}
 	err = self.GetPendingUsage(&pendingRegionUsage, 1)
 	if err == nil && !pendingRegionUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // failure cleanup
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingRegionUsage, &pendingRegionUsage, success)
 	}
 }
 

pkg/compute/tasks/network_create_task.go

@@ -69,7 +69,19 @@ func (self *NetworkCreateTask) OnInit(ctx context.Context, obj db.IStandaloneMod
 		return
 	}
 
-	inet, err := iwire.CreateINetwork(network.Name, prefix.String(), network.Description)
+	opts := cloudprovider.SNetworkCreateOptions{
+		Name: network.Name,
+		Cidr: prefix.String(),
+		Desc: network.Description,
+	}
+
+	provider := wire.GetCloudprovider()
+	opts.ProjectId, err = provider.SyncProject(ctx, self.GetUserCred(), network.ProjectId)
+	if err != nil {
+		log.Errorf("failed to sync project %s for create %s network %s error: %v", network.ProjectId, provider.Provider, network.Name, err)
+	}
+
+	inet, err := iwire.CreateINetwork(&opts)
 	if err != nil {
 		self.taskFailed(ctx, network, "createinetwork", err)
 		return

pkg/compute/tasks/secgroup_group_cache_task.go

@@ -69,7 +69,7 @@ func (self *SecurityGroupCacheTask) OnInit(ctx context.Context, obj db.IStandalo
 
 	self.SetStage("OnCacheSecurityGroupComplete", nil)
 
-	err = region.GetDriver().RequestCacheSecurityGroup(ctx, self.UserCred, region, vpc, secgroup, classic, self)
+	err = region.GetDriver().RequestCacheSecurityGroup(ctx, self.UserCred, region, vpc, secgroup, classic, "", self)
 	if err != nil {
 		self.taskFailed(ctx, secgroup, errors.Wrap(err, "RequestCacheSecgroup"))
 		return

pkg/hostman/diskutils/diskutils.go

@@ -124,7 +124,9 @@ func (d *SKVMGuestDisk) Connect() bool {
 	if pathType == LVM_PATH {
 		d.setupLVMS()
 	} else if pathType == PATH_TYPE_UNKNOWN {
-		if hasLVM, err := d.setupLVMS(); !hasLVM && err == nil {
+		hasLVM, err := d.setupLVMS()
+		// no lvm partition found and has partitions
+		if !hasLVM && err == nil && len(d.partitions) > 0 {
 			d.cacheNonLVMImagePath()
 		}
 	}

pkg/hostman/guestman/guestman.go

@@ -460,6 +460,8 @@ func (m *SGuestManager) GetStatus(sid string) string {
 				timeutils2.AddTimeout(1*time.Second,
 					func() { guest.SyncMirrorJobFailed("Block job missing") })
 				return GUEST_BLOCK_STREAM_FAIL
+			} else {
+				return GUEST_RUNNING
 			}
 		}
 		if guest.IsRunning() {

pkg/hostman/hostinfo/hostinfo.go

@@ -1377,18 +1377,13 @@ func (h *SHostInfo) deployAdminAuthorizedKeys() {
 	}
 
 	sshDir := path.Join("/root", ".ssh")
-	if _, err := os.Stat(sshDir); err != nil {
-		if os.IsNotExist(err) {
-			if err := os.MkdirAll(sshDir, 0755); err != nil {
-				onErr("Create ssh dir %s: %v", sshDir, err)
-			}
-		} else {
-			onErr("Stat %s dir: %v", sshDir, err)
-		}
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", sshDir).Output()
+	if err != nil {
+		onErr("mkdir .ssh failed %s %s", output, err)
 	}
 
 	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("system"), "scope")
+	query.Set("admin", jsonutils.JSONTrue)
 	ret, err := modules.Sshkeypairs.List(h.GetSession(), query)
 	if err != nil {
 		onErr("Get admin sshkey: %v", err)
@@ -1400,15 +1395,23 @@ func (h *SHostInfo) deployAdminAuthorizedKeys() {
 	adminPublicKey, _ := keys.GetString("public_key")
 	pubKeys := &deployapi.SSHKeys{AdminPublicKey: adminPublicKey}
 
+	var oldKeys string
 	authFile := path.Join(sshDir, "authorized_keys")
-	oldKeysBytes, _ := fileutils2.FileGetContents(authFile)
-	oldKeys := string(oldKeysBytes)
-	newKeys := fsdriver.MergeAuthorizedKeys(oldKeys, pubKeys)
-	if err := fileutils2.FilePutContents(authFile, newKeys, false); err != nil {
-		onErr("Write public keys: %v", err)
+	if procutils.NewRemoteCommandAsFarAsPossible("test", "-f", authFile).Run() == nil {
+		output, err := procutils.NewRemoteCommandAsFarAsPossible("cat", authFile).Output()
+		if err != nil {
+			onErr("cat auth file %s %s", output, err)
+		}
+		oldKeys = string(output)
 	}
-	if err := os.Chmod(authFile, 0644); err != nil {
-		onErr("Chmod %s to 0644: %v", authFile, err)
+	newKeys := fsdriver.MergeAuthorizedKeys(oldKeys, pubKeys)
+	if output, err := procutils.NewRemoteCommandAsFarAsPossible(
+		"sh", "-c", fmt.Sprintf("echo '%s' > %s", newKeys, authFile)).Output(); err != nil {
+		onErr("write public keys: %s %s", output, err)
+	}
+	if output, err := procutils.NewRemoteCommandAsFarAsPossible(
+		"chmod", "0644", authFile).Output(); err != nil {
+		onErr("chmod failed %s %s", output, err)
 	}
 	h.onSucc()
 }

pkg/image/models/images.go

@@ -538,7 +538,8 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede
 	// if SImage belong to a guest image, pending quota will not be set.
 	if self.IsGuestImage.IsFalse() {
 		pendingUsage := SQuota{Image: 1}
-		keys := imageCreateInput2QuotaKeys(self.DiskFormat, ownerId)
+		inputDiskFormat, _ := data.GetString("disk_format")
+		keys := imageCreateInput2QuotaKeys(inputDiskFormat, ownerId)
 		pendingUsage.SetKeys(keys)
 		cancelUsage := SQuota{Image: 1}
 		keys = self.GetQuotaKeys()

pkg/mcclient/modules/mod_cloudaccounts.go

@@ -29,6 +29,7 @@ func init() {
 			"Provider", "Brand",
 			"Enable_Auto_Sync", "Sync_Interval_Seconds",
 			"Share_Mode", "is_public", "public_scope",
+			"auto_create_project",
 		},
 		[]string{})
 

pkg/mcclient/modules/mod_cloudproviderregions.go

@@ -29,7 +29,7 @@ func init() {
 			"Enabled", "Sync_Status",
 			"Last_Sync", "Last_Sync_End_At", "Auto_Sync",
 			"last_deep_sync_at",
-			"Sync_Results"},
+		},
 		[]string{},
 		&Cloudproviders,
 		&Cloudregions)

pkg/mcclient/modules/mod_wires.go

@@ -23,7 +23,9 @@ var (
 func init() {
 	Wires = NewComputeManager("wire", "wires",
 		[]string{"ID", "Name", "Bandwidth", "Zone_ID",
-			"Zone", "Networks", "VPC", "VPC_ID", "public_scope"},
+			"Zone", "Networks", "VPC", "VPC_ID", "public_scope",
+			"domain_id",
+		},
 		[]string{})
 
 	registerCompute(&Wires)

pkg/multicloud/aliyun/aliyun.go

@@ -129,7 +129,7 @@ func jsonRequest(client *sdk.Client, domain, apiVersion, apiName string, params
 			}
 			for _, code := range []string{"404 Not Found"} {
 				if strings.Contains(err.Error(), code) {
-					return nil, cloudprovider.ErrNotFound
+					return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error())
 				}
 			}
 			for _, code := range []string{

pkg/multicloud/aliyun/dbinstance.go

@@ -762,26 +762,25 @@ func (rds *SDBInstance) ClosePublicConnection() error {
 }
 
 func (rds *SDBInstance) RecoveryFromBackup(conf *cloudprovider.SDBInstanceRecoveryConfig) error {
-	return rds.region.RecoveryDBInstanceFromBackup(rds.DBInstanceId, conf.BackupId, conf.Databases)
+	if len(conf.OriginDBInstanceExternalId) == 0 {
+		conf.OriginDBInstanceExternalId = rds.DBInstanceId
+	}
+	return rds.region.RecoveryDBInstanceFromBackup(conf.OriginDBInstanceExternalId, rds.DBInstanceId, conf.BackupId, conf.Databases)
 }
 
-func (region *SRegion) RecoveryDBInstanceFromBackup(instanceId string, backupId string, databases map[string]string) error {
-	dbs := []string{}
-	for k, v := range databases {
-		dbs = append(dbs, fmt.Sprintf(`"%s":"%s"`, k, v))
-	}
+func (region *SRegion) RecoveryDBInstanceFromBackup(srcId, destId string, backupId string, databases map[string]string) error {
 	params := map[string]string{
-		"RegionId":     region.RegionId,
-		"DBInstanceId": instanceId,
-		"BackupId":     backupId,
-		"DbNames":      strings.Join(dbs, ","),
+		"RegionId":           region.RegionId,
+		"DBInstanceId":       srcId,
+		"TargetDBInstanceId": destId,
+		"BackupId":           backupId,
+		"DbNames":            jsonutils.Marshal(databases).String(),
 	}
 	_, err := region.rdsRequest("RecoveryDBInstance", params)
 	if err != nil {
 		return errors.Wrap(err, "rdsRequest.RecoveryDBInstance")
 	}
 	return nil
-
 }
 
 func (rds *SDBInstance) GetProjectId() string {

pkg/multicloud/aliyun/dbinstance_backup.go

@@ -293,6 +293,9 @@ func (region *SRegion) waitBackupCreateComplete(instanceId, jobId string) error
 			if job.BackupStatus == "Finished" && job.BackupJobId == jobId {
 				return nil
 			}
+			if job.BackupStatus == "Failed" && job.BackupJobId == jobId {
+				return fmt.Errorf("instance %s backup job %s failed", instanceId, jobId)
+			}
 		}
 		time.Sleep(time.Second * 3)
 	}

pkg/multicloud/aliyun/provider/provider.go

@@ -161,6 +161,22 @@ func (self *SAliyunProvider) GetStorageClasses(regionId string) []string {
 	}
 }
 
+func (self *SAliyunProvider) GetBucketCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
+func (self *SAliyunProvider) GetObjectCannedAcls(regionId string) []string {
+	return []string{
+		string(cloudprovider.ACLPrivate),
+		string(cloudprovider.ACLPublicRead),
+		string(cloudprovider.ACLPublicReadWrite),
+	}
+}
+
 func (self *SAliyunProvider) GetCapabilities() []string {
 	return self.client.GetCapabilities()
 }

pkg/multicloud/aliyun/region.go

@@ -43,6 +43,8 @@ type SRegion struct {
 	RegionId  string
 	LocalName string
 
+	RegionEndpoint string
+
 	izones []cloudprovider.ICloudZone
 
 	ivpcs []cloudprovider.ICloudVpc
@@ -105,7 +107,11 @@ func (self *SRegion) ecsRequest(apiName string, params map[string]string) (jsonu
 	if err != nil {
 		return nil, err
 	}
-	return jsonRequest(client, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.client.debug)
+	endpoint := self.RegionEndpoint
+	if len(endpoint) == 0 {
+		endpoint = "ecs.aliyuncs.com"
+	}
+	return jsonRequest(client, endpoint, ALIYUN_API_VERSION, apiName, params, self.client.debug)
 }
 
 func (self *SRegion) rdsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) {
@@ -763,8 +769,8 @@ func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.I
 	return secgroup, nil
 }
 
-func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) {
-	secgroups, total, err := region.GetSecurityGroups(vpcId, name, []string{}, 0, 0)
+func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) {
+	secgroups, total, err := region.GetSecurityGroups(opts.VpcId, opts.Name, []string{}, 0, 0)
 	if err != nil {
 		return nil, err
 	}

pkg/multicloud/aliyun/shell/dbinstance.go

@@ -76,6 +76,7 @@ func init() {
 
 	type DBInstanceRecoveryOptions struct {
 		ID        string
+		TARGET    string
 		BACKUP    string
 		Databases []string
 	}
@@ -95,7 +96,7 @@ func init() {
 				}
 			}
 		}
-		return cli.RecoveryDBInstanceFromBackup(args.ID, args.BACKUP, databases)
+		return cli.RecoveryDBInstanceFromBackup(args.ID, args.TARGET, args.BACKUP, databases)
 	})
 
 }

pkg/multicloud/aliyun/wire.go

@@ -112,8 +112,8 @@ func (self *SWire) GetBandwidth() int {
 	return 10000
 }
 
-func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) {
-	vswitchId, err := self.zone.region.createVSwitch(self.zone.ZoneId, self.vpc.VpcId, name, cidr, desc)
+func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) {
+	vswitchId, err := self.zone.region.createVSwitch(self.zone.ZoneId, self.vpc.VpcId, opts.Name, opts.Cidr, opts.Desc)
 	if err != nil {
 		log.Errorf("createVSwitch error %s", err)
 		return nil, err