Merge pull request #8283 from ioito/automated-cherry-pick-of-#8281-upstream-release-3.3

Automated cherry pick of #8281: fix: sync huawei bucket project info

cmd/climc/shell/compute/networks.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -360,15 +361,18 @@ func init() {
 
 	type NetworkAddressOptions struct {
 		NETWORK string `help:"id or name of network to query"`
+
+		api.GetNetworkAddressesInput
 	}
 	R(&NetworkAddressOptions{}, "network-addresses", "Query used addresses of network", func(s *mcclient.ClientSession, args *NetworkAddressOptions) error {
-		result, err := modules.Networks.GetSpecific(s, args.NETWORK, "addresses", nil)
+		result, err := modules.Networks.GetSpecific(s, args.NETWORK, "addresses", jsonutils.Marshal(args.GetNetworkAddressesInput))
 		if err != nil {
 			return err
 		}
-		addrList, err := result.GetArray("addresses")
-		if err != nil {
-			return err
+		addrList, _ := result.GetArray("addresses")
+		if addrList == nil {
+			fmt.Println("no result")
+			return nil
 		}
 		listResult := modulebase.ListResult{Data: addrList}
 		printList(&listResult, nil)

cmd/climc/shell/compute/servers.go

@@ -1152,6 +1152,7 @@ func init() {
 		ID               string `help:"ID or name of VM"`
 		DisableIsaSerial string `help:"disable isa serial device" choices:"true|false"`
 		DisablePvpanic   string `help:"disable pvpanic device" choices:"true|false"`
+		DisableUsbKbd    string `help:"disable usb kbd" choices:"true|false"`
 	}
 
 	R(&ServerQemuParams{}, "server-set-qemu-params", "config qemu params", func(s *mcclient.ClientSession,
@@ -1163,6 +1164,9 @@ func init() {
 		if len(opts.DisablePvpanic) > 0 {
 			params.Set("disable_pvpanic", jsonutils.NewString(opts.DisablePvpanic))
 		}
+		if len(opts.DisableUsbKbd) > 0 {
+			params.Set("disable_usb_kbd", jsonutils.NewString(opts.DisableUsbKbd))
+		}
 		result, err := modules.Servers.PerformAction(s, opts.ID, "set-qemu-params", params)
 		if err != nil {
 			return err

cmd/climc/shell/events/events.go

@@ -35,7 +35,10 @@ type BaseEventListOptions struct {
 	Action     []string `help:"Log action"`
 
 	User    string `help:"filter by operator user"`
-	Project string `help:"filter by owner project"`
+	Project string `help:"filter by operator user's project"`
+
+	OwnerProjectIds []string `help:"filter by owner project ids"`
+	OwnerDomainIds  []string `help:"filter by owner domain ids"`
 
 	PagingMarker string `help:"marker for pagination"`
 }
@@ -107,6 +110,12 @@ func doEventList(man modulebase.ResourceManager, s *mcclient.ClientSession, args
 	if len(args.Scope) > 0 {
 		params.Add(jsonutils.NewString(args.Scope), "scope")
 	}
+	if len(args.OwnerProjectIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerProjectIds), "owner_project_ids")
+	}
+	if len(args.OwnerDomainIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerDomainIds), "owner_domain_ids")
+	}
 	if len(args.PagingMarker) > 0 {
 		params.Add(jsonutils.NewString(args.PagingMarker), "paging_marker")
 	}

cmd/climc/shell/image/images.go

@@ -54,6 +54,7 @@ type ImageOptionalOptions struct {
 	Hypervisor         []string `help:"Prefer hypervisor type" choices:"kvm|esxi|baremetal|container|openstack|ctyun"`
 	DiskDriver         string   `help:"Perfer disk driver" choices:"virtio|scsi|pvscsi|ide|sata"`
 	NetDriver          string   `help:"Preferred network driver" choices:"virtio|e1000|vmxnet3"`
+	DisableUsbKbd      bool     `help:"Disable usb keyboard on this image(for hypervisor kvm)"`
 }
 
 func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDict, args ImageOptionalOptions) error {
@@ -131,6 +132,9 @@ func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDi
 	if len(args.Hypervisor) > 0 {
 		params.Add(jsonutils.NewString(strings.Join(args.Hypervisor, ",")), "properties", "hypervisor")
 	}
+	if args.DisableUsbKbd {
+		params.Add(jsonutils.NewString("true"), "properties", "disable_usb_kbd")
+	}
 	return nil
 }
 

cmd/raidcli/main.go

@@ -0,0 +1,167 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	_ "yunion.io/x/onecloud/cmd/raidcli/shell"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/ssh"
+)
+
+type BaseOptions struct {
+	Debug      bool   `help:"debug mode"`
+	Help       bool   `help:"Show help"`
+	Host       string `help:"SSH Host IP" default:"$RAID_HOST" metavar:"RAID_HOST"`
+	Username   string `help:"Username, usually root" default:"$RAID_USERNAME" metavar:"RAID_USERNAME"`
+	Password   string `help:"Password" default:"$RAID_PASSWORD" metavar:"RAID_PASSWORD"`
+	Driver     string `help:"Password" default:"$RAID_DRIVER" metavar:"RAID_DRIVER" choices:"MegaRaid|HPSARaid|Mpt2SAS|MarvelRaid"`
+	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
+}
+
+var (
+	options = &BaseOptions{}
+)
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(options,
+		"raidcli",
+		"Command-line interface to test RAID drivers.",
+		`See "raidcli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient() (raid.IRaidDriver, error) {
+	if len(options.Host) == 0 {
+		return nil, fmt.Errorf("Missing host")
+	}
+
+	if len(options.Username) == 0 {
+		return nil, fmt.Errorf("Missing username")
+	}
+
+	if len(options.Password) == 0 {
+		return nil, fmt.Errorf("Missing password")
+	}
+
+	if len(options.Driver) == 0 {
+		return nil, fmt.Errorf("Missing driver")
+	}
+
+	if options.Debug {
+		raid.Debug = true
+	}
+
+	sshClient, err := ssh.NewClient(
+		options.Host,
+		22,
+		options.Username,
+		options.Password,
+		"",
+	)
+	if err != nil {
+		return nil, fmt.Errorf("ssh client init fail: %s", err)
+	}
+
+	drv := drivers.GetDriver(options.Driver, sshClient)
+	if drv == nil {
+		return nil, fmt.Errorf("not supported driver %s", options.Driver)
+	}
+
+	err = drv.ParsePhyDevs()
+	if err != nil {
+		return nil, fmt.Errorf("parse phyical devices error %s", err)
+	}
+
+	return drv, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	// options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var client raid.IRaidDriver
+				client, e = newClient()
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(client, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/raidcli/shell/raid.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type ShowOptions struct {
+	}
+	shellutils.R(&ShowOptions{}, "show", "show raid info", func(drv raid.IRaidDriver, args *ShowOptions) error {
+		adpts := drv.GetAdapters()
+		for i := range adpts {
+			fmt.Println("Index:", i)
+			fmt.Println(jsonutils.Marshal(adpts[i].GetDevices()).PrettyString())
+		}
+		return nil
+	})
+}

cmd/redfishcli/main.go

@@ -17,12 +17,16 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
+	"strings"
 
 	"yunion.io/x/structarg"
 
 	_ "yunion.io/x/onecloud/cmd/redfishcli/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/redfish"
+	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 	_ "yunion.io/x/onecloud/pkg/util/redfish/loader"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -36,8 +40,12 @@ type BaseOptions struct {
 	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
 }
 
+var (
+	options = &BaseOptions{}
+)
+
 func getSubcommandParser() (*structarg.ArgumentParser, error) {
-	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+	parse, e := structarg.NewArgumentParser(options,
 		"redfishcli",
 		"Command-line interface to redfish API.",
 		`See "redfishcli help COMMAND" for help on a specific command.`)
@@ -62,6 +70,7 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 			return nil
 		}
 	})
+	bmcJnlp()
 	for _, v := range shellutils.CommandTable {
 		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
 		if e != nil {
@@ -71,13 +80,57 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 	return parse, nil
 }
 
+func bmcJnlp() {
+	type BmcGetOptions struct {
+		BRAND string `help:"brand of baremetal" choices:"Lenovo|Huawei|HPE|Dell|Supermicro|Dell6|Dell7|Dell9"`
+		Save  string `help:"save to file"`
+		Debug bool   `help:"turn on debug mode"`
+		Sku   string `help:"sku"`
+		Model string `help:"model"`
+	}
+	shellutils.R(&BmcGetOptions{}, "bmc-jnlp", "Get Java Console JNLP file", func(args *BmcGetOptions) error {
+		ctx := context.Background()
+		parts, err := url.Parse(options.Endpoint)
+		if err != nil {
+			return err
+		}
+		bmc := bmconsole.NewBMCConsole(parts.Hostname(), options.Username, options.Password, args.Debug)
+		var jnlp string
+		switch strings.ToLower(args.BRAND) {
+		case "hp", "hpe":
+			jnlp, err = bmc.GetIloConsoleJNLP(ctx)
+		case "dell", "dell inc.":
+			jnlp, err = bmc.GetIdracConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell6":
+			jnlp, err = bmc.GetIdrac6ConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell7":
+			jnlp, err = bmc.GetIdrac7ConsoleJNLP(ctx, args.Sku, args.Model)
+		case "dell9":
+			jnlp, err = bmc.GetIdrac9ConsoleJNLP(ctx)
+		case "supermicro":
+			jnlp, err = bmc.GetSupermicroConsoleJNLP(ctx)
+		case "lenovo":
+			jnlp, err = bmc.GetLenovoConsoleJNLP(ctx)
+		}
+		if err != nil {
+			return err
+		}
+		if len(args.Save) > 0 {
+			return fileutils2.FilePutContents(args.Save, jnlp, false)
+		} else {
+			fmt.Println(jnlp)
+			return nil
+		}
+	})
+}
+
 func showErrorAndExit(e error) {
 	fmt.Fprintf(os.Stderr, "%s", e)
 	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 
-func newClient(options *BaseOptions) (redfish.IRedfishDriver, error) {
+func newClient() (redfish.IRedfishDriver, error) {
 	if len(options.Endpoint) == 0 {
 		return nil, fmt.Errorf("Missing endpoint")
 	}
@@ -104,7 +157,7 @@ func main() {
 		showErrorAndExit(e)
 	}
 	e = parser.ParseArgs(os.Args[1:], false)
-	options := parser.Options().(*BaseOptions)
+	// options := parser.Options().(*BaseOptions)
 
 	if options.Help {
 		fmt.Print(parser.HelpString())
@@ -122,9 +175,11 @@ func main() {
 			suboptions := subparser.Options()
 			if options.SUBCOMMAND == "help" {
 				e = subcmd.Invoke(suboptions)
+			} else if options.SUBCOMMAND == "bmc-jnlp" {
+				e = subcmd.Invoke(suboptions)
 			} else {
 				var client redfish.IRedfishDriver
-				client, e = newClient(options)
+				client, e = newClient()
 				if e != nil {
 					showErrorAndExit(e)
 				}

go.mod

@@ -131,13 +131,13 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383
+	yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b
+	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11
-	yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8
+	yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6
+	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 
 replace (

go.sum

@@ -1126,8 +1126,8 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383 h1:XEhTD+FGjBjEIWwcY0Ez1lY3oTTzjUOcRgH/i4eFM5A=
-yunion.io/x/jsonutils v0.0.0-20200710080709-7ca71cdb9383/go.mod h1:3U5hspYn9apQHaXKY+fJ4Qgwjey32Kc+Mq+mUe9NamQ=
+yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8 h1:KQVw5EmohJCqBDMdZ3fAcOQNNMIYgVuNCNWNwg4EM3A=
+yunion.io/x/jsonutils v0.0.0-20200814075449-927b118adbd8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
@@ -1136,13 +1136,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 h1:gjqNa2uQzIEXyySsbvVz0Ult
 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b h1:wwPzH8iMRYjtaSM2pBN7nOqeZxR4XW5wdz2VE1KH/Ug=
-yunion.io/x/pkg v0.0.0-20200708071451-a663dc94e16b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b h1:Ov/eKwwNeQje7SggkopExzcCW/GfaUouvXkcY6RUEcM=
-yunion.io/x/pkg v0.0.0-20200713121502-18154120c49b/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
+yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11 h1:srrssomGJLh4OaDaY/1so+2FUBjZxdwQx/1pZ6403YU=
-yunion.io/x/sqlchemy v0.0.0-20200702064124-195445bc4b11/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8 h1:35Q2ce/gI2cuVhsuLS7WsL2K9SxwMEtWTopt/99iYsk=
-yunion.io/x/structarg v0.0.0-20200708124353-ea9a441ecaf8/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6 h1:k7E8Zjwzyk/lo1dDO0jtLbODwYRNv+w/zsUJrKWCXvo=
+yunion.io/x/sqlchemy v0.0.0-20200814052348-164f1f20f8a6/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
+yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/imageutils.go

@@ -0,0 +1,129 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func readImageForm(r *multipart.Reader) (map[string]string, *multipart.Part, error) {
+	params := make(map[string]string)
+
+	maxValueBytes := int64(10 << 20)
+	for {
+		p, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+		filename := p.FileName()
+
+		var b bytes.Buffer
+
+		_, hasContentTypeHeader := p.Header["Content-Type"]
+		if !hasContentTypeHeader && filename == "" {
+			// value, store as string in memory
+			n, err := io.CopyN(&b, p, maxValueBytes+1)
+			if err != nil && err != io.EOF {
+				return nil, nil, err
+			}
+			maxValueBytes -= n
+			if maxValueBytes < 0 {
+				return nil, nil, multipart.ErrMessageTooLarge
+			}
+			params[name] = b.String()
+			continue
+		}
+
+		if name == "image" || name == "file" {
+			return params, p, nil
+		} else {
+			return nil, nil, fmt.Errorf("no file uploaded")
+		}
+	}
+	return nil, nil, fmt.Errorf("empty form")
+}
+
+func imageUploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	params := jsonutils.NewDict()
+
+	name, ok := p["name"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少镜像名称")
+		return
+	}
+	params.Add(jsonutils.NewString(name), "name")
+
+	_imageSize, ok := p["image_size"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少文件信息")
+		return
+	}
+	imageSize, e := strconv.ParseInt(_imageSize, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "文件信息错误")
+		return
+	}
+
+	// add all other params
+	for k, v := range p {
+		if k == "name" || k == "image_size" {
+			continue
+		}
+		params.Add(jsonutils.NewString(v), k)
+	}
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	res, e := modules.Images.Upload(s, params, f, imageSize)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	} else {
+		appsrv.SendJSON(w, res)
+	}
+}
+
+func uploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
+	log.Debugf("%s - %s", method, prefix)
+	hi := appsrv.SHandlerInfo{}
+	hi.SetMethod(method)
+	hi.SetPath(prefix)
+	hi.SetHandler(handler)
+	hi.SetProcessTimeout(6 * time.Hour)
+	hi.SetWorkerManager(GetUploaderWorker())
+	return &hi
+}

pkg/apigateway/handler/misc.go

@@ -20,6 +20,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -52,6 +53,7 @@ const (
 	HOST_IPMI_USERNAME_OPTIONAL = "IPMI用户名"
 	HOST_IPMI_PASSWORD_OPTIONAL = "IPMI密码"
 	HOST_MNG_IP_ADDR_OPTIONAL   = "管理口IP地址"
+	HOST_MNG_MAC_ADDR_OPTIONAL  = "管理口MAC地址"
 )
 
 const (
@@ -62,7 +64,7 @@ const (
 var (
 	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
 	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
-	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_MAC_ADDR_OPTIONAL, HOST_MNG_IP_ADDR_OPTIONAL}
 )
 
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
@@ -90,6 +92,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	app.AddHandler3(uploader)
 	app.AddHandler(GET, prefix+"downloads/<template_id>", FetchAuthToken(h.getDownloadsHandler))
 	app.AddHandler(POST, prefix+"piuploads", FetchAuthToken(h.postPIUploads)) // itsm process instances upload api
+	imageUploader := uploadHandlerInfo("POST", prefix+"/imageutils/upload", FetchAuthToken(imageUploadHandler))
+	app.AddHandler3(imageUploader)
+	s3upload := uploadHandlerInfo(POST, prefix+"s3uploads", FetchAuthToken(h.postS3UploadHandler))
+	app.AddHandler3(s3upload)
 }
 
 func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
@@ -202,7 +208,7 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	i2 := -1
 	for i, title := range rows[0] {
 		switch title {
-		case HOST_MAC:
+		case HOST_MAC, HOST_MNG_MAC_ADDR_OPTIONAL:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")
@@ -509,6 +515,61 @@ func (mh *MiscHandler) postPIUploads(ctx context.Context, w http.ResponseWriter,
 	appsrv.SendJSON(w, resp)
 }
 
+func (mh *MiscHandler) postS3UploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		log.Debugf("postS3UploadHandler.MultipartReader %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		log.Debugf("postS3UploadHandler.readImageForm %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	bucket_id, ok := p["bucket_id"]
+	if !ok {
+		httperrors.MissingParameterError(w, "bucket_id")
+		return
+	}
+
+	key, ok := p["key"]
+	if !ok {
+		httperrors.MissingParameterError(w, "key")
+		return
+	}
+
+	_content_length, ok := p["content_length"]
+	if !ok {
+		httperrors.MissingParameterError(w, "content_length")
+		return
+	}
+
+	content_length, e := strconv.ParseInt(_content_length, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "invalid content_length %s", _content_length)
+		return
+	}
+
+	storage_class, _ := p["storage_class"]
+	acl, _ := p["acl"]
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	meta := http.Header{}
+	meta.Set("Content-Type", "application/octet-stream")
+	e = modules.Buckets.Upload(s, bucket_id, key, f, content_length, storage_class, acl, meta)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.NewDict())
+}
+
 func writeCsv(records [][]string) (bytes.Buffer, error) {
 	var content bytes.Buffer
 	content.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM, 防止office打开后中文乱码

pkg/apis/cloudcommon/proxy/proxy.go

@@ -57,7 +57,7 @@ type ProxySettingResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
+	ProxySettingId string `json:"proxy_setting_id" yunion-deprecated-by:"proxy_setting"`
 }
 
 type ProxySettingTestInput struct {

pkg/apis/cloudid/cloudaccount.go

@@ -22,7 +22,7 @@ type CloudaccountResourceListInput struct {
 	Provider []string `json:"provider"`
 
 	// swagger:ignore
-	CloudaccountId string `json:"cloudaccount_id" "yunion:deprecated-by":"cloudaccount"`
+	CloudaccountId string `json:"cloudaccount_id" yunion-deprecated-by:"cloudaccount"`
 }
 
 type CloudproviderResourceListInput struct {
@@ -30,7 +30,7 @@ type CloudproviderResourceListInput struct {
 	Cloudprovider string `json:"cloudprovider"`
 
 	// swagger:ignore
-	CloudproviderId string `json:"cloudprovider_id" "yunion:deprecated-by":"cloudprovider"`
+	CloudproviderId string `json:"cloudprovider_id" yunion-deprecated-by:"cloudprovider"`
 }
 
 type CloudaccountResourceDetails struct {

pkg/apis/cloudid/cloudgroup.go

@@ -37,7 +37,7 @@ type CloudgroupJointsListInput struct {
 	// 根据公有云用户组过滤资源
 	Cloudgroup string `json:"cloudgroup"`
 	// swagger:ignore
-	CloudgroupId string `json:"cloudgroup_id" "yunion:deprecated-by":"cloudgroup"`
+	CloudgroupId string `json:"cloudgroup_id" yunion-deprecated-by:"cloudgroup"`
 }
 
 type CloudgroupJointBaseUpdateInput struct {

pkg/apis/cloudid/cloudpolicy.go

@@ -56,7 +56,7 @@ type CloudpolicyResourceListInput struct {
 	Cloudpolicy string `json:"cloudpolicy"`
 
 	// swagger:ignore
-	CloudpolicyId string `json:"cloudpolicy_id" "yunion:deprecated-by":"cloudpolicy"`
+	CloudpolicyId string `json:"cloudpolicy_id" yunion-deprecated-by:"cloudpolicy"`
 }
 
 type CloudpolicyResourceDetails struct {

pkg/apis/cloudid/clouduser.go

@@ -162,7 +162,7 @@ type ClouduserResourceListInput struct {
 	Clouduser string `json:"clouduser"`
 
 	// swagger:ignore
-	ClouduserId string `json:"clouduser_id" "yunion:deprecated-by":"clouduser"`
+	ClouduserId string `json:"clouduser_id" yunion-deprecated-by:"clouduser"`
 }
 
 type ClouduserResourceDetails struct {

pkg/apis/compute/api.go

@@ -285,7 +285,7 @@ type ServerConfigs struct {
 	// swagger:ignore
 	// Deprecated
 	// alias for InstanceType
-	Sku string `json:"sku" "yunion:deprecated-by":"instance_type"`
+	Sku string `json:"sku" yunion-deprecated-by:"instance_type"`
 
 	// 虚拟机高可用(创建备机)
 	// default: false
@@ -359,7 +359,7 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	KeypairId string `json:"keypair_id" "yunion:deprecated-by":"keypair"`
+	KeypairId string `json:"keypair_id" yunion-deprecated-by:"keypair"`
 
 	// 秘钥对Id
 	// required: false
@@ -471,7 +471,8 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	OsType string `json:"os_type"`
-
+	// swagger:ignore
+	DisableUsbKbd bool `json:"disable_usb_kbd"`
 	// swagger:ignore
 	OsProfile jsonutils.JSONObject `json:"__os_profile__"`
 	// swagger:ignore

pkg/apis/compute/cachedimage.go

@@ -27,7 +27,7 @@ type CachedImageUncacheImageInput struct {
 }
 
 type CachedimageDetails struct {
-	apis.StandaloneResourceDetails
+	apis.SharableVirtualResourceDetails
 
 	SCachedimage
 

pkg/apis/compute/cachedimages_const.go

@@ -18,7 +18,7 @@ const (
 	CACHED_IMAGE_STATUS_INIT         = "init"
 	CACHED_IMAGE_STATUS_SAVING       = "saving"
 	CACHED_IMAGE_STATUS_CACHING      = "caching"
-	CACHED_IMAGE_STATUS_READY        = "ready"
+	CACHED_IMAGE_STATUS_ACTIVE       = "active"
 	CACHED_IMAGE_STATUS_DELETING     = "deleting"
 	CACHED_IMAGE_STATUS_CACHE_FAILED = "cache_fail"
 

pkg/apis/compute/cloudaccount.go

@@ -66,7 +66,7 @@ type CloudenvResourceListInput struct {
 	Providers []string `json:"providers"`
 	// swagger:ignore
 	// Deprecated
-	Provider []string `json:"provider" "yunion:deprecated-by":"providers"`
+	Provider []string `json:"provider" yunion-deprecated-by:"providers"`
 
 	// 列出指定云平台品牌的资源，一般来说brand和provider相同，除了以上支持的provider之外，还支持以下band
 	//
@@ -77,7 +77,7 @@ type CloudenvResourceListInput struct {
 	Brands []string `json:"brands"`
 	// swagger:ignore
 	// Deprecated
-	Brand []string `json:"brand" "yunion:deprecated-by":"brands"`
+	Brand []string `json:"brand" yunion-deprecated-by:"brands"`
 
 	// 列出指定云环境的资源，支持云环境如下：
 	//
@@ -227,7 +227,7 @@ type CloudaccountListInput struct {
 	// 代理
 	ProxySetting string `json:"proxy_setting"`
 	// swagger:ignore
-	ProxySettingId string `json:"proxy_setting_id" "yunion:deprecated-by":"proxy_setting"`
+	ProxySettingId string `json:"proxy_setting_id" yunion-deprecated-by:"proxy_setting"`
 }
 
 type ProviderProject struct {

pkg/apis/compute/cloudprovider.go

@@ -147,15 +147,15 @@ type CloudproviderResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Manager string `json:"manager" "yunion:deprecated-by":"cloudprovider"`
+	Manager string `json:"manager" yunion-deprecated-by:"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	ManagerId string `json:"manager_id" "yunion:deprecated-by":"cloudprovider"`
+	ManagerId string `json:"manager_id" yunion-deprecated-by:"cloudprovider"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudproviderId string `json:"cloudprovider_id" "yunion:deprecated-by":"cloudprovider"`
+	CloudproviderId string `json:"cloudprovider_id" yunion-deprecated-by:"cloudprovider"`
 }
 
 type ManagedResourceListInput struct {
@@ -169,15 +169,15 @@ type ManagedResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudaccountId string `json:"cloudaccount_id" "yunion:deprecated-by":"cloudaccount"`
+	CloudaccountId string `json:"cloudaccount_id" yunion-deprecated-by:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Account string `json:"account" "yunion:deprecated-by":"cloudaccount"`
+	Account string `json:"account" yunion-deprecated-by:"cloudaccount"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	AccountId string `json:"account_id" "yunion:deprecated-by":"cloudaccount"`
+	AccountId string `json:"account_id" yunion-deprecated-by:"cloudaccount"`
 
 	// 过滤资源，是否为非OneCloud内置私有云管理的资源
 	// default: false

pkg/apis/compute/cloudregions_const.go

@@ -57,6 +57,7 @@ const (
 	CITY_FO_SHAN       = "Foshan"      //佛山
 	CITY_QUAN_ZHOU     = "Quanzhou"    //泉州
 	CITY_NEI_MENG_GU   = "Neimenggu"   //内蒙古
+	CITY_WU_LAN_CHA_BU = "Wulanchabu"  //乌兰察布
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京

pkg/apis/compute/dbinstance.go

@@ -180,7 +180,7 @@ type SDBInstanceRecoveryConfigInput struct {
 	apis.Meta
 
 	// swagger:ignore
-	DBInstancebackup string `json:"dbinstancebackup" "yunion:deprecated-by":"dbinstancebackup_id"`
+	DBInstancebackup string `json:"dbinstancebackup" yunion-deprecated-by:"dbinstancebackup_id"`
 
 	// 备份Id
 	//
@@ -335,7 +335,7 @@ type DBInstanceResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	DBInstanceId string `json:"dbinstance_id" "yunion:deprecated-by":"dbinstance"`
+	DBInstanceId string `json:"dbinstance_id" yunion-deprecated-by:"dbinstance"`
 }
 
 type DBInstanceFilterListInputBase struct {
@@ -350,3 +350,8 @@ type DBInstanceFilterListInput struct {
 
 	VpcFilterListInput
 }
+
+type DBInstanceNetworkListInput struct {
+	apis.VirtualJointResourceBaseListInput
+	DBInstanceFilterListInput
+}

pkg/apis/compute/disk.go

@@ -88,7 +88,7 @@ type SnapshotPolicyResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter disk by snapshotpolicy_id
-	SnapshotpolicyId string `json:"snapshotpolicy_id" "yunion:deprecated-by":"snapshotpolicy"`
+	SnapshotpolicyId string `json:"snapshotpolicy_id" yunion-deprecated-by:"snapshotpolicy"`
 }
 
 type SnapshotPolicyFilterListInput struct {
@@ -113,7 +113,7 @@ type DiskListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk type
-	Type string `json:"type" "yunion:deprecated-by":"disk_type"`
+	Type string `json:"type" yunion-deprecated-by:"disk_type"`
 	// 过滤指定disk_type的磁盘列表，可能的值为：sys, data, swap. volume
 	//
 	// | disk_type值 | 说明 |
@@ -137,13 +137,13 @@ type DiskListInput struct {
 	Template string `json:"template"`
 	// swagger:ignore
 	// Deprecated
-	TemplateId string `json:"template_id" "yunion:deprecated-by":"template"`
+	TemplateId string `json:"template_id" yunion-deprecated-by:"template"`
 
 	// 快照
 	Snapshot string `json:"snapshot"`
 	// swagger:ignore
 	// Deprecated
-	SnapshotId string `json:"snapshot_id" "yunion:deprecated-by":"snapshot"`
+	SnapshotId string `json:"snapshot_id" yunion-deprecated-by:"snapshot"`
 }
 
 type DiskResourceInput struct {
@@ -152,7 +152,7 @@ type DiskResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by disk_id
-	DiskId string `json:"disk_id" "yunion:deprecated-by":"disk"`
+	DiskId string `json:"disk_id" yunion-deprecated-by:"disk"`
 }
 
 type DiskFilterListInputBase struct {

pkg/apis/compute/dnsrecord.go

@@ -14,5 +14,11 @@
 
 package compute
 
+import "yunion.io/x/onecloud/pkg/apis"
+
+type DnsRecordCreateInput struct {
+	apis.AdminSharableVirtualResourceBaseCreateInput
+}
+
 type DnsRecordUpdateInput struct {
 }

pkg/apis/compute/elasticcache.go

@@ -53,7 +53,7 @@ type ELasticcacheResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ElasticcacheId string `json:"elasticcache_id" "yunion:deprecated-by":"elasticcache"`
+	ElasticcacheId string `json:"elasticcache_id" yunion-deprecated-by:"elasticcache"`
 }
 
 type ElasticcacheFilterListInput struct {

pkg/apis/compute/elasticip.go

@@ -21,23 +21,22 @@ type SElasticipCreateInput struct {
 
 	// 区域名称或Id, 建议使用Id
 	// 在指定区域内创建弹性公网ip
-	Cloudregion string `json:"cloudregion"`
+	CloudregionId string `json:"cloudregion_id"`
+	// swagger:ignore
+	Cloudregion string `json:"cloudregion" yunion-deprecated-by:"cloudregion_id"`
+	// swagger:ignore
+	Region string `json:"region" yunion-deprecated-by:"cloudregion_id"`
+	// swagger:ignore
+	RegionId string `json:"region_id" yunion-deprecated-by:"cloudregion_id"`
 
 	// swagger:ignore
-	Region string
+	Cloudprovider string `json:"cloudprovider" yunion-deprecated-by:"manager_id"`
 	// swagger:ignore
-	RegionId string
-	// swagger:ignore
-	CloudregionId string
-
+	Manager string `json:"manager" yunion-deprecated-by:"manager_id"`
 	// 子订阅Id, 建议使用Id
 	// 使用指定子订阅创建弹性公网ip
 	// 弹性公网ip和虚拟机在同一区域同一子订阅底下，才可以进行绑定操作
-	Cloudprovider string `json:"cloudprovider"`
-	// swagger:ignore
-	Manager string
-	// swagger:ignore
-	ManagerId string
+	ManagerId string `json:"manager_id"`
 
 	// 计费类型: 流量或带宽
 	//
@@ -63,11 +62,11 @@ type SElasticipCreateInput struct {
 
 	Mode string `json:"mode"`
 
-	// 子网名称或Id
-	// 私有云创建此参数必传,例如Openstack, ZStack
-	Network string `json:"network"`
 	// swagger:ignore
-	NetworkId string
+	Network string `json:"network" yunion-deprecated-by:"network_id"`
+	// 子网Id
+	// 私有云创建此参数必传,例如Openstack, ZStack
+	NetworkId string `json:"network_id"`
 }
 
 type ElasticipDetails struct {

pkg/apis/compute/geo_input.go

@@ -24,15 +24,15 @@ type CloudregionResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	CloudregionId string `json:"cloudregion_id" "yunion:deprecated-by":"cloudregion"`
+	CloudregionId string `json:"cloudregion_id" yunion-deprecated-by:"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	Region string `json:"region" "yunion:deprecated-by":"cloudregion"`
+	Region string `json:"region" yunion-deprecated-by:"cloudregion"`
 	// swagger:ignore
 	// Deprecated
 	// description: this param will be deprecate at 3.0
-	RegionId string `json:"region_id" "yunion:deprecated-by":"cloudregion"`
+	RegionId string `json:"region_id" yunion-deprecated-by:"cloudregion"`
 }
 
 type RegionalFilterListInput struct {
@@ -134,5 +134,5 @@ type ZoneResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ZoneId string `json:"zone_id" "yunion:deprecated-by":"zone"`
+	ZoneId string `json:"zone_id" yunion-deprecated-by:"zone"`
 }

pkg/apis/compute/globalvpc.go

@@ -42,7 +42,7 @@ type GlobalVpcResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	GlobalvpcId string `json:"globalvpc_id" "yunion:deprecated-by":"globalvpc"`
+	GlobalvpcId string `json:"globalvpc_id" yunion-deprecated-by:"globalvpc"`
 }
 
 type GlobalVpcResourceListInput struct {

pkg/apis/compute/guests.go

@@ -28,13 +28,13 @@ type ServerListInput struct {
 
 	HostFilterListInput
 
-	NetworkFilterListInput `"yunion:ambiguous-prefix":"vpc_"`
+	NetworkFilterListInput `yunion-ambiguous-prefix:"vpc_"`
 
 	billing.BillingResourceListInput
 
 	GroupFilterListInput
 	SecgroupFilterListInput
-	//DiskFilterListInput `"yunion:ambiguous-prefix":"storage_"`
+	//DiskFilterListInput `yunion-ambiguous-prefix:"storage_"`
 	ScalingGroupFilterListInput
 
 	// 只列出裸金属主机
@@ -64,7 +64,7 @@ type ServerListInput struct {
 	// 列出可以挂载磁盘的主机
 	AttachableServersForDisk string `json:"attachable_servers_for_disk"`
 	// Deprecated:列出可以挂载磁盘的主机
-	Disk string `json:"disk" "yunion:deprecated-by":"attachable_servers_for_disk"`
+	Disk string `json:"disk" yunion-deprecated-by:"attachable_servers_for_disk"`
 
 	// 按主机资源类型进行排序
 	// enum: shared,prepaid,dedicated
@@ -285,15 +285,15 @@ type ServerResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	ServerId string `json:"server_id" "yunion:deprecated-by":"server"`
+	ServerId string `json:"server_id" yunion-deprecated-by:"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	Guest string `json:"guest" "yunion:deprecated-by":"server"`
+	Guest string `json:"guest" yunion-deprecated-by:"server"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by guest Id
-	GuestId string `json:"guest_id" "yunion:deprecated-by":"server"`
+	GuestId string `json:"guest_id" yunion-deprecated-by:"server"`
 }
 
 type ServerFilterListInput struct {

pkg/apis/compute/host.go

@@ -141,7 +141,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率
@@ -232,7 +232,7 @@ type HostResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by host_id
-	HostId string `json:"host_id" "yunion:deprecated-by":"host"`
+	HostId string `json:"host_id" yunion-deprecated-by:"host"`
 }
 
 type HostRegisterMetadata struct {

pkg/apis/compute/hostjoin.go

@@ -21,7 +21,7 @@ type HostJointResourceDetailsBase struct {
 	Host string `json:"host"`
 	// 裸金属服务器名称
 	// Deprecated
-	Baremetal string `json:"baremetal" "yunion:deprecated-by":"host"`
+	Baremetal string `json:"baremetal" yunion-deprecated-by:"host"`
 }
 
 type HostJointResourceDetails struct {

pkg/apis/compute/input.go

@@ -49,7 +49,7 @@ type KeypairListInput struct {
 }
 
 type CachedimageListInput struct {
-	apis.StandaloneResourceListInput
+	apis.SharableVirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 
 	ManagedResourceListInput
@@ -144,7 +144,7 @@ type GuestTemplateFilterListInput struct {
 	GuestTemplate string `json:"guest_template"`
 	// swagger:ignore
 	// Deprecated
-	GuestTemplateId string `json:"guest_template_id" "yunion:deprecated-by":"guest_template"`
+	GuestTemplateId string `json:"guest_template_id" yunion-deprecated-by:"guest_template"`
 }
 
 type ServiceCatalogListInput struct {

pkg/apis/compute/instance_group.go

@@ -50,7 +50,7 @@ type GroupResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by instance group Id
-	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
+	GroupId string `json:"group_id" yunion-deprecated-by:"group"`
 }
 
 type GroupFilterListInput struct {

pkg/apis/compute/loadbalancer.go

@@ -201,7 +201,10 @@ type LoadbalancerResourceInfo struct {
 	// 可用区ID
 	ZoneId string `json:"zone_id"`
 
-	ZoneResourceInfoBase
+	ZoneResourceInfo
+
+	// cloud provider info
+	ManagedResourceInfo
 }
 
 type LoadbalancerResourceInput struct {
@@ -210,7 +213,7 @@ type LoadbalancerResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	LoadbalancerId string `json:"loadbalancer_id" "yunion:deprecated-by":"loadbalancer"`
+	LoadbalancerId string `json:"loadbalancer_id" yunion-deprecated-by:"loadbalancer"`
 }
 
 type LoadbalancerFilterListInput struct {

pkg/apis/compute/loadbalanceracl.go

@@ -35,7 +35,7 @@ type LoadbalancerAclResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	Acl string `json:"acl" "yunion:deprecated-by":"acl_id"`
+	Acl string `json:"acl" yunion-deprecated-by:"acl_id"`
 }
 
 type LoadbalancerAclFilterListInput struct {

pkg/apis/compute/loadbalancerbackendgroup.go

@@ -39,7 +39,7 @@ type LoadbalancerBackendGroupResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	BackendGroupId string `json:"backend_group_id" "yunion:deprecated-by":"backend_group"`
+	BackendGroupId string `json:"backend_group_id" yunion-deprecated-by:"backend_group"`
 }
 
 type LoadbalancerBackendGroupFilterListInput struct {

pkg/apis/compute/loadbalancercertificate.go

@@ -32,7 +32,7 @@ type LoadbalancerCertificateResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	CertificateId string `json:"certificate_id" "yunion:deprecated-by":"certificate"`
+	CertificateId string `json:"certificate_id" yunion-deprecated-by:"certificate"`
 }
 
 type LoadbalancerCertificateFilterListInput struct {

pkg/apis/compute/loadbalancercluster.go

@@ -45,7 +45,7 @@ type LoadbalancerClusterResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	ClusterId string `json:"cluster_id" "yunion:deprecated-by":"cluster"`
+	ClusterId string `json:"cluster_id" yunion-deprecated-by:"cluster"`
 }
 
 type LoadbalancerClusterFilterListInput struct {

pkg/apis/compute/loadbalancerlistener.go

@@ -46,7 +46,7 @@ type LoadbalancerListenerResourceInput struct {
 	// 负载均衡监听器ID
 	// swagger:ignore
 	// Deprecated
-	ListenerId string `json:"listener_id" "yunion:deprecated-by":"listener"`
+	ListenerId string `json:"listener_id" yunion-deprecated-by:"listener"`
 }
 
 type LoadbalancerListenerFilterListInput struct {

pkg/apis/compute/natgateway.go

@@ -89,7 +89,7 @@ type NatGatewayResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	NatgatewayId string `json:"natgateway_id" "yunion:deprecated-by":"natgateway"`
+	NatgatewayId string `json:"natgateway_id" yunion-deprecated-by:"natgateway"`
 }
 
 type NatGatewayFilterListInput struct {

pkg/apis/compute/network.go

@@ -24,7 +24,7 @@ type WireResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// fitler by wire id
-	WireId string `json:"wire_id" "yunion:deprecated-by":"wire"`
+	WireId string `json:"wire_id" yunion-deprecated-by:"wire"`
 }
 
 type WireFilterListBase struct {
@@ -47,7 +47,7 @@ type NetworkResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by networkId
-	NetworkId string `json:"network_id" "yunion:deprecated-by":"network"`
+	NetworkId string `json:"network_id" yunion-deprecated-by:"network"`
 }
 
 type NetworkFilterListBase struct {
@@ -300,3 +300,13 @@ type NetworkUpdateInput struct {
 	// 分配策略
 	AllocPolicy string `json:"alloc_policy"`
 }
+
+type GetNetworkAddressesInput struct {
+	// 获取资源的范围，例如 project|domain|system
+	Scope string `json:"scope"`
+}
+
+type GetNetworkAddressesOutput struct {
+	// IP子网地址记录
+	Addresses []SNetworkAddress `json:"addresses"`
+}

pkg/apis/compute/schedtag.go

@@ -45,7 +45,7 @@ type SchedtagResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by schedtag_id
-	SchedtagId string `json:"schedtag_id" "yunion:deprecated-by":"schedtag"`
+	SchedtagId string `json:"schedtag_id" yunion-deprecated-by:"schedtag"`
 }
 
 type SchedtagFilterListInput struct {
@@ -69,7 +69,7 @@ type SchedtagListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by type, alias for resource_type
-	Type string `json:"type" "yunion:deprecated-by":"resource_type"`
+	Type string `json:"type" yunion-deprecated-by:"resource_type"`
 
 	DefaultStrategy []string `json:"default_strategy"`
 }

pkg/apis/compute/secgroup.go

@@ -194,7 +194,7 @@ type SecgroupResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by secgroup_id
-	SecgroupId string `json:"secgroup_id" "yunion:deprecated-by":"secgroup"`
+	SecgroupId string `json:"secgroup_id" yunion-deprecated-by:"secgroup"`
 
 	// 模糊匹配安全组规则名称
 	SecgroupName string `json:"secgroup_name"`
@@ -248,3 +248,6 @@ type GuestsecgroupDetails struct {
 	// 安全组名称
 	Secgroup string `json:"secgroup"`
 }
+
+type SecurityGroupPurgeInput struct {
+}

pkg/apis/compute/storage_const.go

@@ -144,7 +144,7 @@ type StorageResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by storage_id
-	StorageId string `json:"storage_id" "yunion:deprecated-by":"storage"`
+	StorageId string `json:"storage_id" yunion-deprecated-by:"storage"`
 }
 
 type StorageFilterListInputBase struct {

pkg/apis/compute/storagecachedimage.go

@@ -48,13 +48,13 @@ type StoragecachedimageListInput struct {
 	Storagecache string `json:"storagecache"`
 	// Deprecated
 	// swagger:ignore
-	StoragecacheId string `json:"storagecache_id" "yunion:deprecated-by":"storagecache"`
+	StoragecacheId string `json:"storagecache_id" yunion-deprecated-by:"storagecache"`
 
 	// 以镜像缓存过滤
 	Cachedimage string `json:"cachedimage"`
 	// Deprecated
 	// swagger:ignore
-	CachedimageId string `json:"cachedimage_id" "yunion:deprecated-by":"cachedimage"`
+	CachedimageId string `json:"cachedimage_id" yunion-deprecated-by:"cachedimage"`
 
 	// 镜像状态
 	Status []string `json:"status"`

pkg/apis/compute/vpc.go

@@ -90,7 +90,7 @@ type VpcResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by vpc Id
-	VpcId string `json:"vpc_id" "yunion:deprecated-by":"vpc"`
+	VpcId string `json:"vpc_id" yunion-deprecated-by:"vpc"`
 
 	// Vpc外网访问模式
 	ExternalAccessMode string `json:"external_access_mode"`

pkg/apis/compute/vpcs_ovn.go

@@ -32,6 +32,34 @@ var (
 	}
 )
 
+const (
+	sVpcInterCidr    = "100.65.0.0/17"
+	sVpcInterExtCidr = "100.65.0.0/30"
+	VpcInterExtMask  = 30
+	sVpcInterExtIP1  = "100.65.0.1"
+	sVpcInterExtIP2  = "100.65.0.2"
+	VpcInterExtMac1  = "ee:ee:ee:ee:ee:f0"
+	VpcInterExtMac2  = "ee:ee:ee:ee:ee:f1"
+)
+
+var (
+	vpcInterCidr   netutils.IPV4Prefix
+	vpcInterExtIP1 netutils.IPV4Addr
+	vpcInterExtIP2 netutils.IPV4Addr
+)
+
+func VpcInterCidr() netutils.IPV4Prefix {
+	return vpcInterCidr
+}
+
+func VpcInterExtIP1() netutils.IPV4Addr {
+	return vpcInterExtIP1
+}
+
+func VpcInterExtIP2() netutils.IPV4Addr {
+	return vpcInterExtIP2
+}
+
 const (
 	sVpcMappedCidr      = "100.64.0.0/17"
 	VpcMappedIPMask     = 17
@@ -92,6 +120,10 @@ func init() {
 		return v
 	}
 
+	vpcInterCidr = mp(netutils.NewIPV4Prefix(sVpcInterCidr))
+	vpcInterExtIP1 = mi(netutils.NewIPV4Addr(sVpcInterExtIP1))
+	vpcInterExtIP2 = mi(netutils.NewIPV4Addr(sVpcInterExtIP2))
+
 	vpcMappedCidr = mp(netutils.NewIPV4Prefix(sVpcMappedCidr))
 	vpcMappedGatewayIP = mi(netutils.NewIPV4Addr(sVpcMappedGatewayIP))
 

pkg/apis/identity/identityprovider.go

@@ -73,7 +73,7 @@ type IdentityProviderCreateInput struct {
 	TargetDomain string `json:"target_domain"`
 	// swagger:ignore
 	// Deprecated
-	TargetDomainId string `json:"target_domain_id" "yunion:deprecated-by":"target_domain"`
+	TargetDomainId string `json:"target_domain_id" yunion-deprecated-by:"target_domain"`
 
 	// 新建域的时候是否自动新建第一个项目
 	AutoCreateProject *bool `json:"auto_create_project"`

pkg/apis/identity/input.go

@@ -49,22 +49,22 @@ type ProjectFilterListInput struct {
 	ProjectDomain string `json:"project_domain"`
 	// swagger:ignore
 	// Deprecated
-	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project-domain"`
+	ProjectDomainId string `json:"project_domain_id" yunion-deprecated-by:"project-domain"`
 
 	// 以项目（ID或Name）过滤列表结果
 	Project string `json:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by project_id
-	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
+	ProjectId string `json:"project_id" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant
-	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
+	Tenant string `json:"tenant" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// filter by tenant_id
-	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
+	TenantId string `json:"tenant_id" yunion-deprecated-by:"project"`
 }
 
 type UserFilterListInput struct {
@@ -79,7 +79,7 @@ type UserFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by user_id
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 }
 
 type GroupFilterListInput struct {
@@ -94,7 +94,7 @@ type GroupFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by group_id
-	GroupId string `json:"group_id" "yunion:deprecated-by":"group"`
+	GroupId string `json:"group_id" yunion-deprecated-by:"group"`
 }
 
 type RoleFilterListInput struct {
@@ -109,7 +109,7 @@ type RoleFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by role_id
-	RoleId string `json:"role_id" "yunion:deprecated-by":"role"`
+	RoleId string `json:"role_id" yunion-deprecated-by:"role"`
 }
 
 type ServiceFilterListInput struct {
@@ -121,7 +121,7 @@ type ServiceFilterListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// filter by service_id
-	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
+	ServiceId string `json:"service_id" yunion-deprecated-by:"service"`
 
 	// 以服务名称排序
 	OrderByService string `json:"order_by_service"`
@@ -325,7 +325,7 @@ type RegionFilterListInput struct {
 	Region string `json:"region"`
 	// swagger:ignore
 	// Deprecated
-	RegionId string `json:"region_id" "yunion:deprecated-by":"region"`
+	RegionId string `json:"region_id" yunion-deprecated-by:"region"`
 }
 
 type RegionListInput struct {

pkg/apis/image/consts.go

@@ -50,6 +50,7 @@ const (
 	IMAGE_IS_READONLY         = "is_readonly"
 	IMAGE_PARTITION_TYPE      = "partition_type"
 	IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init"
+	IMAGE_DISABLE_USB_KBD     = "disable_usb_kbd"
 
 	IMAGE_STATUS_UPDATING = "updating"
 )

pkg/apis/image/image.go

@@ -88,3 +88,12 @@ type ImageCreateInput struct {
 	// 镜像属性
 	Properties map[string]string `json:"properties"`
 }
+
+type ImageUpdateStatusInput struct {
+	apis.Meta
+
+	// 镜像状态
+	Status string `json:"status"`
+	// 更新镜像状态原因
+	Reason string `json:"reason"`
+}

pkg/apis/input.go

@@ -21,15 +21,15 @@ type DomainizedResourceInput struct {
 
 	// swagger:ignore
 	// Deprecated
-	Domain string `json:"domain" "yunion:deprecated-by":"project_domain"`
+	Domain string `json:"domain" yunion-deprecated-by:"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Project domain Id filter, alias for project_domain
-	ProjectDomainId string `json:"project_domain_id" "yunion:deprecated-by":"project_domain"`
+	ProjectDomainId string `json:"project_domain_id" yunion-deprecated-by:"project_domain"`
 	// swagger:ignore
 	// Deprecated
 	// Domain Id filter, alias for project_domain
-	DomainId string `json:"domain_id" "yunion:deprecated-by":"project_domain"`
+	DomainId string `json:"domain_id" yunion-deprecated-by:"project_domain"`
 }
 
 type ProjectizedResourceInput struct {
@@ -39,15 +39,15 @@ type ProjectizedResourceInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by project_id, alias for project
-	ProjectId string `json:"project_id" "yunion:deprecated-by":"project"`
+	ProjectId string `json:"project_id" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant ID or Name, alias for project
-	Tenant string `json:"tenant" "yunion:deprecated-by":"project"`
+	Tenant string `json:"tenant" yunion-deprecated-by:"project"`
 	// swagger:ignore
 	// Deprecated
 	// Filter by tenant_id, alias for project
-	TenantId string `json:"tenant_id" "yunion:deprecated-by":"project"`
+	TenantId string `json:"tenant_id" yunion-deprecated-by:"project"`
 }
 
 type DomainizedResourceCreateInput struct {
@@ -75,6 +75,13 @@ type SharableVirtualResourceCreateInput struct {
 	SharableResourceBaseCreateInput
 }
 
+type AdminSharableVirtualResourceBaseCreateInput struct {
+	SharableVirtualResourceCreateInput
+
+	// 记录
+	Records string `json:"records"`
+}
+
 type StatusDomainLevelUserResourceCreateInput struct {
 	StatusDomainLevelResourceCreateInput
 

pkg/apis/list.go

@@ -50,7 +50,7 @@ type ProjectizedResourceListInput struct {
 	OrderByProject string `json:"order_by_project"`
 	// swagger:ignore
 	// Deprecated
-	OrderByTenant string `json:"order_by_tenant" "yunion:deprecated-by":"order_by_project"`
+	OrderByTenant string `json:"order_by_tenant" yunion-deprecated-by:"order_by_project"`
 }
 
 type StatusDomainLevelUserResourceListInput struct {
@@ -61,7 +61,7 @@ type StatusDomainLevelUserResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by userId
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 }
 
 type UserResourceListInput struct {
@@ -78,7 +78,7 @@ type UserResourceListInput struct {
 	// swagger:ignore
 	// Deprecated
 	// Filter by userId
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 }
 
 type ModelBaseListInput struct {

pkg/apis/output.go

@@ -155,11 +155,11 @@ type ProjectizedResourceInfo struct {
 
 	// 资源归属项目的ID(向后兼容别名）
 	// Deprecated
-	TenantId string `json:"project_id" "yunion:deprecated-by":"tenant_id"`
+	TenantId string `json:"project_id" yunion-deprecated-by:"tenant_id"`
 
 	// 资源归属项目的名称（向后兼容别名）
 	// Deprecated
-	Tenant string `json:"project" "yunion:deprecated-by":"tenant"`
+	Tenant string `json:"project" yunion-deprecated-by:"tenant"`
 }
 
 type ScopedResourceBaseInfo struct {

pkg/apis/yunionconf/input.go

@@ -28,14 +28,14 @@ type ParameterListInput struct {
 
 	// Deprecated
 	// swagger:ignore
-	ServiceId string `json:"service_id" "yunion:deprecated-by":"service"`
+	ServiceId string `json:"service_id" yunion-deprecated-by:"service"`
 
 	// 用户名称或ID
 	User string `json:"user"`
 
 	// Deprecated
 	// swagger:ignore
-	UserId string `json:"user_id" "yunion:deprecated-by":"user"`
+	UserId string `json:"user_id" yunion-deprecated-by:"user"`
 
 	// filter by name
 	Name []string `json:"name"`

pkg/baremetal/manager.go

@@ -61,6 +61,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/dhcp"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/redfish"
@@ -1398,7 +1399,15 @@ func (b *SBaremetalInstance) DoRedfishPowerOn() error {
 	b.ClearSSHConfig()
 	redfishApi := b.GetRedfishCli(ctx)
 	if redfishApi != nil {
-		return redfishApi.Reset(ctx, "On")
+		err := redfishApi.Reset(ctx, "On")
+		if err != nil {
+			if httputils.ErrorCode(err) == 409 {
+				log.Warningf("redfishApi.Reset On fail %s", err)
+			} else {
+				return errors.Wrap(err, "redfishApi.Reset On")
+			}
+		}
+		return nil
 	}
 	return fmt.Errorf("Baremetal %s redfishApi is nil", b.GetId())
 }
@@ -1950,9 +1959,11 @@ func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error)
 	case "hp", "hpe":
 		return bmc.GetIloConsoleJNLP(ctx)
 	case "dell", "dell inc.":
-		return bmc.GetIdracConsoleJNLP(ctx, "", "")
+		return bmc.GetIdracConsoleJNLP(ctx, b.GetSerialNumber(), b.GetModel())
 	case "supermicro":
 		return bmc.GetSupermicroConsoleJNLP(ctx)
+	case "lenovo":
+		return bmc.GetLenovoConsoleJNLP(ctx)
 	}
 	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
 }

pkg/baremetal/pxe/dhcp.go

@@ -269,6 +269,9 @@ func (req *dhcpRequest) findNetworkConf(session *mcclient.ClientSession, filterU
 		params.Add(jsonutils.NewString(
 			fmt.Sprintf("guest_dhcp.endswith(',%s')", req.RelayAddr)),
 			"filter.3")
+		params.Add(jsonutils.NewString(
+			fmt.Sprintf("guest_dhcp.equals(%s)", req.RelayAddr)),
+			"filter.4")
 		params.Add(jsonutils.JSONTrue, "filter_any")
 	}
 	params.Add(jsonutils.JSONTrue, "is_on_premise")

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -20,9 +20,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/pkg/utils"
@@ -75,7 +74,7 @@ func (dev *MegaRaidPhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalSto
 }
 
 func (dev *MegaRaidPhyDev) GetSize() int64 {
-	return dev.sector * int64(dev.block) / 1024 / 1024 // MB
+	return dev.sector * dev.block / 1024 / 1024 // MB
 }
 
 func (dev *MegaRaidPhyDev) parseLine(line string) bool {
@@ -124,7 +123,7 @@ func (dev *MegaRaidPhyDev) parseLine(line string) bool {
 		if err != nil {
 			log.Errorf("parse logical sector size error: %v", err)
 			dev.block = 512
-		} else {
+		} else if block > 0 {
 			dev.block = int64(block)
 		}
 	default:
@@ -189,6 +188,7 @@ type MegaRaidAdaptor struct {
 	raid         *MegaRaid
 	devs         []*MegaRaidPhyDev
 	sn           string
+	name         string
 	busNumber    string
 	deviceNumber string
 	funcNumber   string
@@ -209,6 +209,10 @@ func NewMegaRaidAdaptor(index int, raid *MegaRaid) (*MegaRaidAdaptor, error) {
 	return adapter, nil
 }
 
+func (adapter MegaRaidAdaptor) key() string {
+	return adapter.name + adapter.sn
+}
+
 func (adapter *MegaRaidAdaptor) fillInfo() error {
 	cmd := GetCommand("-AdpAllInfo", fmt.Sprintf("-a%d", adapter.index))
 	ret, err := adapter.remoteRun(cmd)
@@ -223,10 +227,12 @@ func (adapter *MegaRaidAdaptor) fillInfo() error {
 		switch key {
 		case "Serial No":
 			adapter.sn = val
+		case "Product Name":
+			adapter.name = val
 		}
 	}
-	if len(adapter.sn) == 0 {
-		return errors.New("Not found Serial No")
+	if len(adapter.key()) == 0 {
+		return errors.Error("Not found Serial No and Product Name")
 	}
 	return adapter.fillPCIInfo()
 }
@@ -267,7 +273,7 @@ func (adapter *MegaRaidAdaptor) fillPCIInfo() error {
 		}
 	}
 	if len(adapter.busNumber) == 0 || len(adapter.deviceNumber) == 0 || len(adapter.funcNumber) == 0 {
-		return errors.New("Not found bus number")
+		return errors.Error("Not found bus number")
 	}
 	pciDir := fmt.Sprintf("/sys/bus/pci/devices/0000:%s:%s.%s/", adapter.busNumber, adapter.deviceNumber, adapter.funcNumber)
 	cmd = raiddrivers.GetCommand("ls", pciDir, "|", "grep", "host")
@@ -601,18 +607,24 @@ func (adapter *MegaRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage
 
 type StorcliAdaptor struct {
 	Controller int
-	SN         string
+	sn         string
+	name       string
 }
 
 func newStorcliAdaptor() *StorcliAdaptor {
 	return &StorcliAdaptor{
 		Controller: -1,
-		SN:         "",
+		sn:         "",
+		name:       "",
 	}
 }
 
+func (a StorcliAdaptor) key() string {
+	return a.name + a.sn
+}
+
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.SN != ""
+	return a.Controller >= 0 && a.key() != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {
@@ -626,13 +638,15 @@ func (a *StorcliAdaptor) parseLine(l string) {
 	case "Controller":
 		a.Controller, _ = strconv.Atoi(val)
 	case "Serial Number":
-		a.SN = val
+		a.sn = val
+	case "Product Name":
+		a.name = val
 	}
 }
 
 func (raid *MegaRaid) GetStorcliAdaptor() (map[string]*StorcliAdaptor, error) {
 	ret := make(map[string]*StorcliAdaptor)
-	cmd := GetCommand2("/call", "show", "|", "grep", "-iE", `'^(Controller|Serial Number)\s='`)
+	cmd := GetCommand2("/call", "show", "|", "grep", "-iE", `'^(Controller|Product Name|Serial Number)\s='`)
 	lines, err := raid.term.Run(cmd)
 	if err != nil {
 		return nil, errors.Wrap(err, "Get storcli adapter")
@@ -641,7 +655,7 @@ func (raid *MegaRaid) GetStorcliAdaptor() (map[string]*StorcliAdaptor, error) {
 	for _, l := range lines {
 		adapter.parseLine(l)
 		if adapter.isComplete() {
-			ret[adapter.SN] = adapter
+			ret[adapter.key()] = adapter
 			adapter = newStorcliAdaptor()
 		}
 	}
@@ -656,9 +670,9 @@ func (adapter *MegaRaidAdaptor) storcliCtrlIndex() (int, error) {
 	if err != nil {
 		return -1, errors.Wrap(err, "Get all Storcli adaptor")
 	}
-	storAdap, ok := storcliAdaps[adapter.sn]
+	storAdap, ok := storcliAdaps[adapter.key()]
 	if !ok {
-		return -1, errors.Errorf("Not found storcli adaptor by SN %q", adapter.sn)
+		return -1, errors.Errorf("Not found storcli adaptor by SN %q", adapter.key())
 	}
 	return storAdap.Controller, nil
 }
@@ -914,6 +928,9 @@ func (raid *MegaRaid) ParsePhyDevs() error {
 	if err != nil {
 		return fmt.Errorf("List raid disk error: %v", err)
 	}
+	if raiddrivers.Debug {
+		log.Debugf("-PDList -aALL: %s", ret)
+	}
 	err = raid.parsePhyDevs(ret)
 	if err != nil {
 		return fmt.Errorf("parse physical disk device error: %v", err)

pkg/baremetal/utils/raid/raid.go

@@ -28,6 +28,10 @@ import (
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
+var (
+	Debug bool
+)
+
 const (
 	MODULE_MEGARAID = "megaraid_sas"
 	MODULE_HPSA     = "hpsa"

pkg/cloudcommon/agent/agent.go

@@ -239,6 +239,7 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
 	if agent.IAgent().GetAgentType() != string(api.AgentTypeEsxi) {
 		params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
 	}
+	params.Add(jsonutils.NewString(agent.IAgent().GetZoneName()), "zone_id")
 	params.Add(jsonutils.NewString(agent.IAgent().GetAgentType()), "agent_type")
 	ret, err := modules.Baremetalagents.List(session, params)
 	if err != nil {

pkg/cloudcommon/db/adminsharablevirtual.go

@@ -51,7 +51,22 @@ func (manager *SAdminSharableVirtualResourceBaseManager) GetIAdminSharableVirtua
 	return manager.GetVirtualObject().(IAdminSharableVirtualModelManager)
 }
 
-func (manager *SAdminSharableVirtualResourceBaseManager) ValidateCreateData(man IAdminSharableVirtualModelManager, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (manager *SAdminSharableVirtualResourceBaseManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input apis.AdminSharableVirtualResourceBaseCreateInput,
+) (apis.AdminSharableVirtualResourceBaseCreateInput, error) {
+	var err error
+	input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData")
+	}
+	return input, nil
+}
+
+func (manager *SAdminSharableVirtualResourceBaseManager) ValidateRecordsData(man IAdminSharableVirtualModelManager, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	records, err := man.ParseInputInfo(data)
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/db_dispatcher.go

@@ -1351,7 +1351,7 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 		ret := make([]sCreateResult, len(multiData))
 		for i := range multiData {
 			var model IModel
-			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i)
+			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i+1)
 			if err == nil {
 				ret[i] = sCreateResult{model: model, err: nil}
 			} else {

pkg/cloudcommon/db/domain.go

@@ -106,9 +106,16 @@ func (manager *SDomainizedResourceBaseManager) ListItemFilter(
 	query apis.DomainizedResourceListInput,
 ) (*sqlchemy.SQuery, error) {
 	if len(query.ProjectDomains) > 0 {
+		// make sure ids are not utf8 string
+		idList := make([]string, 0)
+		for _, pid := range query.ProjectDomains {
+			if !stringutils2.IsUtf8(pid) {
+				idList = append(idList, pid)
+			}
+		}
 		tenants := TenantCacheManager.GetDomainQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(tenants.Field("id"), query.ProjectDomains),
+			sqlchemy.In(tenants.Field("id"), idList),
 			sqlchemy.In(tenants.Field("name"), query.ProjectDomains),
 		)).SubQuery()
 		q = q.In("domain_id", subq)

pkg/cloudcommon/db/opslog.go

@@ -16,6 +16,7 @@ package db
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strconv"
 	"strings"
@@ -464,28 +465,7 @@ func (manager *SOpsLogManager) ListItemFilter(
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
 ) (*sqlchemy.SQuery, error) {
-	/*userStrs := jsonutils.GetQueryStringArray(query, "user")
-	if len(userStrs) > 0 {
-		for i := range userStrs {
-			usrObj, err := DefaultUserFetcher(ctx, userStrs[i])
-			if err != nil {
-				if err == sql.ErrNoRows {
-					return nil, httperrors.NewResourceNotFoundError2("user", userStrs[i])
-				} else if err == sqlchemy.ErrDuplicateEntry {
-					return nil, httperrors.NewDuplicateNameError("user", userStrs[i])
-				} else {
-					return nil, httperrors.NewGeneralError(err)
-				}
-			}
-			userStrs[i] = usrObj.GetId()
-		}
-		if len(userStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("user_id"), userStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("user_id"), userStrs))
-		}
-	}
-	projStrs := jsonutils.GetQueryStringArray(query, "project")
+	projStrs := jsonutils.GetQueryStringArray(query, "owner_project_ids")
 	if len(projStrs) > 0 {
 		for i := range projStrs {
 			projObj, err := DefaultProjectFetcher(ctx, projStrs[i])
@@ -498,12 +478,23 @@ func (manager *SOpsLogManager) ListItemFilter(
 			}
 			projStrs[i] = projObj.GetId()
 		}
-		if len(projStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("owner_tenant_id"), projStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+		q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+	}
+	domainStrs := jsonutils.GetQueryStringArray(query, "owner_domain_ids")
+	if len(domainStrs) > 0 {
+		for i := range domainStrs {
+			domainObj, err := DefaultDomainFetcher(ctx, domainStrs[i])
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("domain", domainStrs[i])
+				} else {
+					return nil, httperrors.NewGeneralError(err)
+				}
+			}
+			domainStrs[i] = domainObj.GetId()
 		}
-	}*/
+		q = q.Filter(sqlchemy.In(q.Field("owner_domain_id"), domainStrs))
+	}
 	objTypes := jsonutils.GetQueryStringArray(query, "obj_type")
 	if len(objTypes) > 0 {
 		if len(objTypes) == 1 {

pkg/cloudcommon/db/project.go

@@ -101,9 +101,16 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SDomainizedResourceBaseManager.ListItemFilter")
 	}
 	if len(query.Projects) > 0 {
+		// make sure ids are not utf8 string
+		idList := make([]string, 0)
+		for _, pid := range query.Projects {
+			if !stringutils2.IsUtf8(pid) {
+				idList = append(idList, pid)
+			}
+		}
 		tenants := TenantCacheManager.GetTenantQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(tenants.Field("id"), query.Projects),
+			sqlchemy.In(tenants.Field("id"), idList),
 			sqlchemy.In(tenants.Field("name"), query.Projects),
 		)).SubQuery()
 		q = q.In("tenant_id", subq)

pkg/cloudcommon/db/rbac.go

@@ -43,7 +43,7 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeSystem:
 		requireScope = rbacutils.ScopeSystem
 	case rbacutils.ScopeDomain:
-		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId()) {
+		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() && action == policy.PolicyActionGet) {
 			requireScope = rbacutils.ScopeUser
 		} else if ownerId != nil && objOwnerId != nil && (ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() || objOwnerId.GetProjectDomainId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeDomain

pkg/cloudcommon/db/taskman/subtasks.go

@@ -46,7 +46,7 @@ type SSubTask struct {
 	Stage     string `width:"64" charset:"ascii" nullable:"false" primary:"true"` // Column(VARCHAR(64, charset='ascii'), nullable=False, primary_key=True)
 	SubtaskId string `width:"36" charset:"ascii" nullable:"false" primary:"true"` // Column(VARCHAR(36, charset='ascii'), nullable=False, primary_key=True)
 	Status    string `width:"36" charset:"ascii" nullable:"false" default:"init"` // Column(VARCHAR(36, charset='ascii'), nullable=False, default=SUBTASK_INIT)
-	Result    string `length:"medium" charset:"ascii" nullable:"true"`            // Column(MEDIUMTEXT(charset='ascii'), nullable=True)
+	Result    string `length:"medium" charset:"utf8" nullable:"true"`             // Column(MEDIUMTEXT(charset='ascii'), nullable=True)
 }
 
 func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SSubTask {

pkg/cloudcommon/notifyclient/notify.go

@@ -126,7 +126,7 @@ func RawNotify(recipientId []string, isGroup bool, channel notify.TNotifyChannel
 	msg.Topic = topic
 	body, _ := getContent(event, "content", channel, data)
 	if len(body) == 0 {
-		body = data.String()
+		body, _ = data.GetString()
 	}
 	msg.Msg = body
 	// log.Debugf("send notification %s %s", topic, body)

pkg/cloudprovider/resources.go

@@ -156,7 +156,7 @@ type ICloudZone interface {
 }
 
 type ICloudImage interface {
-	ICloudResource
+	IVirtualResource
 
 	Delete(ctx context.Context) error
 	GetIStoragecache() ICloudStoragecache
@@ -173,6 +173,7 @@ type ICloudImage interface {
 	GetImageFormat() string
 	GetCreatedAt() time.Time
 	UEFI() bool
+	GetPublicScope() rbacutils.TRbacScope
 }
 
 type ICloudStoragecache interface {
@@ -487,6 +488,7 @@ type ICloudHostNetInterface interface {
 	GetIpAddr() string
 	GetMtu() int32
 	GetNicType() string
+	GetBridge() string
 }
 
 type ICloudLoadbalancer interface {

pkg/compute/guestdrivers/base.go

@@ -314,6 +314,10 @@ func (self *SBaseGuestDriver) IsWindowsUserDataTypeNeedEncode() bool {
 	return false
 }
 
+func (self *SBaseGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return true
+}
+
 func (self *SBaseGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_CONFIG
 }

pkg/compute/guestdrivers/esxi.go

@@ -172,7 +172,7 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 	if img.ImageType != cloudprovider.CachedImageTypeSystem {
 		return desc
 	}
-	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_READY).SubQuery()
+	sciSubQ := models.StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", templateId).Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE).SubQuery()
 	scQ := models.StoragecacheManager.Query().In("id", sciSubQ)
 	storageCaches := make([]models.SStoragecache, 0, 1)
 	err = db.FetchModelObjects(models.StoragecacheManager, scQ, &storageCaches)
@@ -368,3 +368,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/guestdrivers/managedvirtual.go

@@ -435,8 +435,8 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte
 	}
 
 	iVM, err := func() (cloudprovider.ICloudVM, error) {
-		lockman.LockObject(ctx, host)
-		defer lockman.ReleaseObject(ctx, host)
+		lockman.LockObject(ctx, guest)
+		defer lockman.ReleaseObject(ctx, guest)
 
 		iVM, err := ihost.CreateVM(&desc)
 		if err != nil {

pkg/compute/guestdrivers/qcloud.go

@@ -226,6 +226,10 @@ func (self *SQcloudGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_RUNNING
 }
 
+func (self *SQcloudGuestDriver) IsSupportdDcryptPasswordFromSecretKey() bool {
+	return false
+}
+
 func (self *SQcloudGuestDriver) GetUserDataType() string {
 	return cloudprovider.CLOUD_SHELL
 }

pkg/compute/hostdrivers/managedvirtual.go

@@ -182,7 +182,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		osType, _ := params.GetString("properties", "os_type")
 
 		scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId, "")
-		if scimg.Status != api.CACHED_IMAGE_STATUS_READY {
+		if scimg.Status != api.CACHED_IMAGE_STATUS_ACTIVE {
 			scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host")
 		}
 		iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, "")
@@ -208,7 +208,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
 		if err := snapshot.Delete(); err != nil {
 			log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err)
 		}
-		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_READY, "")
+		scimg.SetStatus(task.GetUserCred(), api.CACHED_IMAGE_STATUS_ACTIVE, "")
 		return result, nil
 	})
 	return nil

pkg/compute/models/buckets.go

@@ -199,7 +199,13 @@ func (manager *SBucketManager) newFromCloudBucket(
 
 	stats := extBucket.GetStats()
 	bucket.SizeBytes = stats.SizeBytes
+	if bucket.SizeBytes < 0 {
+		bucket.SizeBytes = 0
+	}
 	bucket.ObjectCnt = stats.ObjectCount
+	if bucket.ObjectCnt < 0 {
+		bucket.ObjectCnt = 0
+	}
 
 	limit := extBucket.GetLimit()
 	limitSupport := extBucket.LimitSupport()
@@ -259,7 +265,13 @@ func (bucket *SBucket) syncWithCloudBucket(
 	diff, err := db.UpdateWithLock(ctx, bucket, func() error {
 		stats := extBucket.GetStats()
 		bucket.SizeBytes = stats.SizeBytes
+		if bucket.SizeBytes < 0 {
+			bucket.SizeBytes = 0
+		}
 		bucket.ObjectCnt = stats.ObjectCount
+		if bucket.ObjectCnt < 0 {
+			bucket.ObjectCnt = 0
+		}
 
 		if !statsOnly {
 			limit := extBucket.GetLimit()
@@ -704,11 +716,7 @@ func (bucket *SBucket) GetDetailsObjects(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	prefix := input.Prefix
 	isRecursive := false
@@ -782,11 +790,7 @@ func (bucket *SBucket) PerformTempUrl(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second)
 	if err != nil {
@@ -830,11 +834,7 @@ func (bucket *SBucket) PerformMakedir(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	_, err = cloudprovider.GetIObject(iBucket, key+"/")
@@ -901,11 +901,7 @@ func (bucket *SBucket) PerformDelete(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 	ok := jsonutils.NewDict()
 	results := modulebase.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
@@ -965,11 +961,7 @@ func (bucket *SBucket) PerformUpload(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, appParams.Request.Header)
@@ -1007,7 +999,7 @@ func (bucket *SBucket) PerformUpload(
 		if inc.SizeBytes < 0 {
 			inc.SizeBytes = 0
 		}
-	} else if err == cloudprovider.ErrNotFound {
+	} else if errors.Cause(err) == cloudprovider.ErrNotFound {
 		// new upload
 		inc.SizeBytes = sizeBytes
 		inc.ObjectCount = 1
@@ -1173,11 +1165,7 @@ func (bucket *SBucket) PerformSync(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, statsOnly)
@@ -1221,11 +1209,7 @@ func (bucket *SBucket) GetDetailsAcl(
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return output, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return output, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return output, errors.Wrap(err, "GetIBucket")
 	}
 	objKey := input.Key
 	var acl cloudprovider.TBucketACLType
@@ -1234,7 +1218,7 @@ func (bucket *SBucket) GetDetailsAcl(
 	} else {
 		object, err := cloudprovider.GetIObject(iBucket, objKey)
 		if err != nil {
-			if err == cloudprovider.ErrNotFound {
+			if errors.Cause(err) == cloudprovider.ErrNotFound {
 				return output, httperrors.NewNotFoundError("object %s not found", objKey)
 			} else {
 				return output, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err)
@@ -1309,11 +1293,7 @@ func (bucket *SBucket) PerformLimit(
 
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, errors.Wrap(err, "GetIBucket")
 	}
 
 	err = iBucket.SetLimit(limit)
@@ -1426,11 +1406,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 	}
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, nil, errors.Wrap(err, "GetIBucket")
 	}
 	objects := make([]cloudprovider.ICloudObject, 0)
 	for _, key := range input.Key {
@@ -1443,7 +1419,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 		} else {
 			object, err := cloudprovider.GetIObject(iBucket, key)
 			if err != nil {
-				if err == cloudprovider.ErrNotFound {
+				if errors.Cause(err) == cloudprovider.ErrNotFound {
 					return nil, nil, httperrors.NewResourceNotFoundError("object %s not found", key)
 				} else {
 					return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObject error %s", err)

pkg/compute/models/cachedimages.go

@@ -37,11 +37,12 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 type SCachedimageManager struct {
-	db.SStandaloneResourceBaseManager
+	db.SSharableVirtualResourceBaseManager
 	db.SExternalizedResourceBaseManager
 }
 
@@ -49,7 +50,7 @@ var CachedimageManager *SCachedimageManager
 
 func init() {
 	CachedimageManager = &SCachedimageManager{
-		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
 			SCachedimage{},
 			"cachedimages_tbl",
 			"cachedimage",
@@ -60,7 +61,7 @@ func init() {
 }
 
 type SCachedimage struct {
-	db.SStandaloneResourceBase
+	db.SSharableVirtualResourceBase
 	db.SExternalizedResourceBase
 
 	// 镜像大小单位: Byte
@@ -100,10 +101,10 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {
 	if cnt > 0 {
 		return httperrors.NewNotEmptyError("The image has been cached on storages")
 	}
-	if self.GetStatus() == "active" && !self.isReferenceSessionExpire() {
+	if self.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !self.isReferenceSessionExpire() {
 		return httperrors.NewConflictError("the image reference session has not been expired!")
 	}
-	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
 func (self *SCachedimage) isReferenceSessionExpire() bool {
@@ -194,32 +195,44 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
-	imgId, _ := info.GetString("id")
-	if len(imgId) == 0 {
+	img := struct {
+		Id          string
+		Size        int64
+		Name        string
+		Status      string
+		IsPublic    bool
+		ProjectId   string `json:"tenant_id"`
+		PublicScope string
+	}{}
+	err := info.Unmarshal(&img)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Unmarshal %s", info.String())
+	}
+	if len(img.Id) == 0 {
 		return nil, fmt.Errorf("invalid image info")
 	}
+	if len(img.Name) == 0 {
+		img.Name = img.Id
+	}
 
 	imageCache := SCachedimage{}
 	imageCache.SetModelManager(manager, &imageCache)
 
-	size, _ := info.Int("size")
-	name, _ := info.GetString("name")
-	if len(name) == 0 {
-		name = imgId
-	}
-
-	name, err := db.GenerateName(manager, nil, name)
+	img.Name, err = db.GenerateName(manager, nil, img.Name)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "db.GenerateName(%s)", img.Name)
 	}
 
-	err = manager.RawQuery().Equals("id", imgId).First(&imageCache)
+	err = manager.RawQuery().Equals("id", img.Id).First(&imageCache)
 	if err != nil {
 		if err == sql.ErrNoRows { // insert
-			imageCache.Id = imgId
-			imageCache.Name = name
-			imageCache.Size = size
+			imageCache.Id = img.Id
+			imageCache.Name = img.Name
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 
 			err = manager.TableSpec().Insert(ctx, &imageCache)
@@ -230,13 +243,16 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 
 			return &imageCache, nil
 		} else {
-			log.Errorf("fetching image cache (%s) failed: %s", imgId, err)
+			log.Errorf("fetching image cache (%s) failed: %s", img.Id, err)
 			return nil, err
 		}
 	} else { // update
 		diff, err := db.Update(&imageCache, func() error {
-			imageCache.Size = size
+			imageCache.Size = img.Size
 			imageCache.Info = info
+			imageCache.Status = img.Status
+			imageCache.IsPublic = img.IsPublic
+			imageCache.PublicScope = img.PublicScope
 			imageCache.LastSync = timeutils.UtcNow()
 			if imageCache.Deleted == true {
 				imageCache.Deleted = false
@@ -323,16 +339,15 @@ func (manager *SCachedimageManager) FetchCustomizeColumns(
 	isList bool,
 ) []api.CachedimageDetails {
 	rows := make([]api.CachedimageDetails, len(objs))
-	stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	virtRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range rows {
 		ci := objs[i].(*SCachedimage)
 		rows[i] = api.CachedimageDetails{
-			StandaloneResourceDetails: stdRows[i],
-			Status:                    ci.GetStatus(),
-			OsType:                    ci.GetOSType(),
-			OsDistribution:            ci.GetOSDistribution(),
-			OsVersion:                 ci.GetOSVersion(),
-			Hypervisor:                ci.GetHypervisor(),
+			SharableVirtualResourceDetails: virtRows[i],
+			OsType:                         ci.GetOSType(),
+			OsDistribution:                 ci.GetOSDistribution(),
+			OsVersion:                      ci.GetOSVersion(),
+			Hypervisor:                     ci.GetHypervisor(),
 		}
 		rows[i].CachedCount, _ = ci.getStoragecacheCount()
 	}
@@ -372,7 +387,7 @@ func (self *SCachedimage) PerformUncacheImage(ctx context.Context, userCred mccl
 }
 
 func (self *SCachedimage) addRefCount() {
-	if self.GetStatus() != "active" {
+	if self.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE {
 		return
 	}
 	_, err := db.Update(self, func() error {
@@ -397,7 +412,7 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 		Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))).
 		Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)).
-		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_READY)).
+		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE)).
 		Filter(sqlchemy.Equals(host.Field("status"), api.HOST_STATUS_RUNNING)).
 		Filter(sqlchemy.IsTrue(host.Field("enabled"))).
 		Filter(sqlchemy.Equals(host.Field("host_status"), api.HOST_ONLINE)).
@@ -455,12 +470,17 @@ func (self *SCachedimage) canDeleteLastCache() bool {
 	return false
 }
 
-func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error {
+func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		self.Name = image.GetName()
 		self.Size = image.GetSizeByte()
 		self.ExternalId = image.GetGlobalId()
 		self.ImageType = image.GetImageType()
+		self.PublicScope = string(image.GetPublicScope())
+		self.Status = image.GetStatus()
+		if image.GetPublicScope() == rbacutils.ScopeSystem {
+			self.IsPublic = true
+		}
 		self.UEFI = tristate.NewFromBool(image.UEFI())
 		sImage := cloudprovider.CloudImage2Image(image)
 		self.Info = jsonutils.Marshal(&sImage)
@@ -468,10 +488,12 @@ func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mccli
 		return nil
 	})
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
+
+	SyncCloudProject(userCred, self, ownerId, image, managerId)
 	return err
 }
 
-func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) (*SCachedimage, error) {
+func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) (*SCachedimage, error) {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -491,12 +513,21 @@ func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userC
 	cachedImage.LastSync = time.Now().UTC()
 	cachedImage.ImageType = image.GetImageType()
 	cachedImage.ExternalId = image.GetGlobalId()
+	cachedImage.Status = image.GetStatus()
+	cachedImage.PublicScope = string(image.GetPublicScope())
+	switch image.GetPublicScope() {
+	case rbacutils.ScopeNone:
+	default:
+		cachedImage.IsPublic = true
+	}
 
 	err = manager.TableSpec().Insert(ctx, &cachedImage)
 	if err != nil {
 		return nil, err
 	}
 
+	SyncCloudProject(userCred, &cachedImage, ownerId, image, managerId)
+
 	return &cachedImage, nil
 }
 
@@ -529,7 +560,7 @@ func (image *SCachedimage) requestRefreshExternalImage(ctx context.Context, user
 		log.Errorf("iCache.GetIImageById fail %s", err)
 		return nil, err
 	}
-	err = image.syncWithCloudImage(ctx, userCred, iImage)
+	err = image.syncWithCloudImage(ctx, userCred, nil, iImage, "")
 	if err != nil {
 		log.Errorf("image.syncWithCloudImage fail %s", err)
 		return nil, err
@@ -549,7 +580,7 @@ func (image *SCachedimage) getValidStoragecache() []SStoragecache {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 
 	caches := make([]SStoragecache, 0)
 	err := db.FetchModelObjects(StoragecacheManager, q, &caches)
@@ -589,7 +620,7 @@ func (image *SCachedimage) GetUsableZoneIds() ([]string, error) {
 	q = q.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 	q = q.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("cachedimage_id"), image.Id))
-	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+	q = q.Filter(sqlchemy.Equals(storagecacheimages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 	q = q.Filter(sqlchemy.Equals(q.Field("status"), api.ZONE_ENABLE))
 
 	result := []string{}
@@ -632,6 +663,10 @@ func (manager *SCachedimageManager) ListItemFilter(
 	query api.CachedimageListInput,
 ) (*sqlchemy.SQuery, error) {
 	var err error
+	q, err = manager.SSharableBaseResourceManager.ListItemFilter(ctx, q, userCred, query.SharableResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
+	}
 
 	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
 		cachedImages := CachedimageManager.Query().SubQuery()
@@ -647,9 +682,9 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
 	}
 
-	q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
 	}
 
 	q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput)
@@ -667,7 +702,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -682,7 +717,7 @@ func (manager *SCachedimageManager) ListItemFilter(
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_READY))
+		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
 		return subq
 	})
 	if err != nil {
@@ -704,9 +739,9 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneResourceListInput)
+	q, err = manager.SSharableVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields")
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.OrderByExtraFields")
 	}
 
 	return q, nil
@@ -715,10 +750,44 @@ func (manager *SCachedimageManager) OrderByExtraFields(
 func (manager *SCachedimageManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
 	var err error
 
-	q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
+	q, err = manager.SSharableVirtualResourceBaseManager.QueryDistinctExtraField(q, field)
 	if err == nil {
 		return q, nil
 	}
 
 	return q, httperrors.ErrNotFound
 }
+
+func (manager *SCachedimageManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
+	q, err := manager.SSharableVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SSharableVirtualResourceBaseManager.L.ListItemExportKeys")
+	}
+	return q, nil
+}
+
+func (manager *SCachedimageManager) InitializeData() error {
+	images := []SCachedimage{}
+	q := manager.Query().IsNullOrEmpty("tenant_id")
+	err := db.FetchModelObjects(manager, q, &images)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range images {
+		_, err := db.Update(&images[i], func() error {
+			images[i].IsPublic = true
+			images[i].PublicScope = string(rbacutils.ScopeSystem)
+			images[i].ProjectId = "system"
+			if len(images[i].ExternalId) > 0 {
+				images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE
+			} else {
+				images[i].Status = images[i].GetStatus()
+			}
+			return nil
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update(%s)", images[i].Id)
+		}
+	}
+	return nil
+}

pkg/compute/models/capabilities.go

@@ -615,20 +615,6 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 		region = zone.GetRegion()
 	}
 
-	/*vpcQuery := VpcManager.Query()
-	if len(domainId) > 0 {
-		ownerId := &db.SOwnerId{DomainId: domainId}
-		vpcQuery = VpcManager.FilterByOwner(vpcQuery, ownerId, rbacutils.ScopeDomain)
-	}
-	vpcs := vpcQuery.SubQuery()*/
-
-	wireQuery := WireManager.Query().Equals("vpc_id", api.DEFAULT_VPC_ID)
-	if len(domainId) > 0 {
-		ownerId := &db.SOwnerId{DomainId: domainId}
-		wireQuery = WireManager.FilterByOwner(wireQuery, ownerId, rbacutils.ScopeDomain)
-	}
-	wires := wireQuery.SubQuery()
-
 	networks := NetworkManager.Query().SubQuery()
 
 	q := networks.Query()
@@ -639,9 +625,11 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 			q = q.IsFalse("is_public")
 		}
 	}
-	q = q.Join(wires, sqlchemy.Equals(networks.Field("wire_id"), wires.Field("id")))
 
-	if region != nil {
+	if zone != nil && !utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
+		wires := WireManager.Query("id").Equals("zone_id", zone.Id)
+		q = q.In("wire_id", wires.SubQuery())
+	} else if region != nil {
 		if utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
 			wires := WireManager.Query().SubQuery()
 			vpcs := VpcManager.Query().SubQuery()
@@ -651,28 +639,14 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 			q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq))
 		} else {
 			subq := getRegionZoneSubq(region)
-			q = q.Filter(sqlchemy.In(wires.Field("zone_id"), subq))
+			wires := WireManager.Query("id").In("zone_id", subq)
+			q = q.In("wire_id", wires.SubQuery())
 		}
 	}
-	if zone != nil && !utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
-		q = q.Filter(sqlchemy.Equals(wires.Field("zone_id"), zone.Id))
-	}
+
 	if len(domainId) > 0 {
 		ownerId := &db.SOwnerId{DomainId: domainId}
 		q = NetworkManager.FilterByOwner(q, ownerId, rbacutils.ScopeDomain)
-		/*subq := getDomainManagerSubq(domainId)
-		q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
-		q = q.Filter(sqlchemy.OR(
-			sqlchemy.In(vpcs.Field("manager_id"), subq),
-			sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")),
-		))
-		if isPublic.Bool() {
-			q = q.Filter(sqlchemy.OR(
-				sqlchemy.Equals(q.Field("public_scope"), rbacutils.ScopeSystem),
-				sqlchemy.AND(
-					sqlchemy.Equals(q.Field("public_scope"), rbacutils.ScopeDomain),
-					sqlchemy.Equals(q.Field("domain_id"), domainId))))
-		}*/
 	}
 	if len(serverType) > 0 {
 		q = q.Filter(sqlchemy.Equals(networks.Field("server_type"), serverType))

pkg/compute/models/cloudaccounts.go

@@ -48,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -355,7 +356,14 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		return output, httperrors.NewNotSupportedError("not support for cloudaccount with provider '%s'", input.Provider)
 	}
 	// validate first
-	input.CloudaccountCreateInput, err = scm.ValidateCreateData(ctx, userCred, userCred, query, input.CloudaccountCreateInput)
+	ownerId, err := scm.FetchOwnerId(ctx, jsonutils.Marshal(input))
+	if err != nil {
+		return output, errors.Wrap(err, "FetchOwnerId in PerformPrepareNets")
+	}
+	if ownerId == nil {
+		ownerId = userCred
+	}
+	input.CloudaccountCreateInput, err = scm.ValidateCreateData(ctx, userCred, ownerId, query, input.CloudaccountCreateInput)
 	if err != nil {
 		return output, err
 	}
@@ -417,9 +425,11 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 	if err != nil {
 		return output, errors.Wrap(err, "provider.GetOnPremiseIRegion")
 	}
-	hosts, err := iregion.GetIHosts()
+	// hack
+	client := iregion.(*esxi.SESXiClient)
+	hostIps, simpleVms, err := client.HostVmIPs(ctx)
 	if err != nil {
-		return output, errors.Wrap(err, "unable to get hosts")
+		return output, errors.Wrap(err, "unable to fetch ips of hosts and vms")
 	}
 	// fetch networks
 	networks := make([][]SNetwork, len(wires))
@@ -430,20 +440,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		networks[i] = nets
 	}
-	// fetch host ips
+
 	// key of ipHosts is host's ip
-	ipHosts := make(map[netutils.IPV4Addr]cloudprovider.ICloudHost, len(hosts))
-	for i := range hosts {
-		ip := hosts[i].GetAccessIp()
-		if len(ip) == 0 {
-			log.Errorf("unable to get accessip of host %q", hosts[i].GetName())
-			continue
-		}
+	ipHosts := make(map[netutils.IPV4Addr]string, len(hostIps))
+	for name, ip := range hostIps {
 		addr, err := netutils.NewIPV4Addr(ip)
 		if err != nil {
 			return output, err
 		}
-		ipHosts[addr] = hosts[i]
+		ipHosts[addr] = name
 	}
 	// Find suitable wire and the network containing the Host IP in suitable wire.
 	var (
@@ -490,15 +495,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Give the suggested network configuration for the Host IP that does not have a corresponding suitable network.
 	noNetHostIP := make([]netutils.IPV4Addr, 0, len(ipHosts))
-	for hip, host := range ipHosts {
+	for ip, name := range ipHosts {
 		rnet := api.CAHostNet{
-			Name: host.GetName(),
-			IP:   host.GetAccessIp(),
+			Name: name,
+			IP:   ip.String(),
 		}
-		if net, ok := suitableNetworks[hip]; ok {
+		if net, ok := suitableNetworks[ip]; ok {
 			rnet.SuitableNetwork = net.GetId()
 		} else {
-			noNetHostIP = append(noNetHostIP, hip)
+			noNetHostIP = append(noNetHostIP, ip)
 		}
 		output.Hosts = append(output.Hosts, rnet)
 	}
@@ -515,7 +520,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 
 	// Find the suitable network containing the VM IP in Project 'input.Project', and if not, give the corresponding suggested network configuration in this project.
 	project := input.Project
-	nets := []*SNetwork{}
+	excludeNets := []*SNetwork{}
 	var allNets []SNetwork
 	if suitableWire != nil {
 		allNets, err = suitableWire.getNetworks(userCred, rbacutils.ScopeSystem)
@@ -524,85 +529,164 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		}
 		for i := range allNets {
 			if allNets[i].ProjectId == project {
-				nets = append(nets, &allNets[i])
+				excludeNets = append(excludeNets, &allNets[i])
 			}
 		}
 	}
-	// find suitable network
-	vms := make([]cloudprovider.ICloudVM, 0, len(hosts))
-	for _, host := range hosts {
-		vs, err := host.GetIVMs()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable to get VMs of host %q", host.GetName())
-		}
-		vms = append(vms, vs...)
+
+	type vm struct {
+		FakeID int
+		IP     netutils.IPV4Addr
+		Name   string
 	}
-	ipVMs := make(map[netutils.IPV4Addr]int, len(vms))
-	for i, vm := range vms {
-		nics, err := vm.GetINics()
-		if err != nil {
-			return output, errors.Wrapf(err, "unable get nics of vm %q", vm.GetName())
-		}
-		for _, nic := range nics {
-			ipStr := nic.GetIP()
-			if len(ipStr) == 0 {
-				continue
-			}
-			ip, err := netutils.NewIPV4Addr(ipStr)
-			if err != nil {
-				return output, errors.Wrapf(err, "unable to new IPV4Addr for ip %q", nic.GetIP())
-			}
-			ipVMs[ip] = i
-		}
-		output.Guests = append(output.Guests, api.CAGuestNet{
-			Name:   vm.GetName(),
-			IPNets: make([]api.CAIPNet, 0, 1),
-		})
-	}
-	suitableNetworks = make(map[netutils.IPV4Addr]*SNetwork)
-	if len(nets) != 0 {
-		for vip := range ipVMs {
-			for i := range nets {
-				ipRange := nets[i].GetIPRange()
-				if !ipRange.Contains(vip) {
-					continue
+	vms := make([]vm, 0, len(simpleVms))
+	var nip netutils.IPV4Addr
+	guestMap := make(map[int]*api.CAGuestNet)
+	for i := range simpleVms {
+		id := i
+		if len(simpleVms[i].IPs) == 0 {
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   simpleVms[i].Name,
+					IPNets: []api.CAIPNet{},
 				}
-				suitableNetworks[vip] = nets[i]
-				break
 			}
 		}
-	}
-
-	noNetVMIPs := make([]netutils.IPV4Addr, 0, len(ipVMs))
-	for vip, i := range ipVMs {
-		ipnet := api.CAIPNet{
-			IP: vip.String(),
+		for _, ip := range simpleVms[i].IPs {
+			nip, err = netutils.NewIPV4Addr(ip)
+			if err != nil {
+				return output, err
+			}
+			vms = append(vms, vm{
+				FakeID: i,
+				IP:     nip,
+				Name:   simpleVms[i].Name,
+			})
 		}
-		if net, ok := suitableNetworks[vip]; ok {
-			ipnet.SuitableNetwork = net.GetId()
-		} else {
-			noNetVMIPs = append(noNetVMIPs, vip)
-		}
-		output.Guests[i].IPNets = append(output.Guests[ipVMs[vip]].IPNets, ipnet)
 	}
+	// sort vms via vm's ip
+	sort.Slice(vms, func(i, j int) bool {
+		return vms[i].IP < vms[j].IP
+	})
 
-	// excludedIRs is to prevent network conflicts
-	excludedIRs := make([]netutils.IPV4AddrRange, len(allNets))
-	for i := range excludedIRs {
-		excludedIRs[i] = allNets[i].GetIPRange()
+	excludeNets = make([]*SNetwork, 0, len(allNets)+len(output.HostSuggestedNetworks))
+	for i := range allNets {
+		excludeNets = append(excludeNets, &allNets[i])
 	}
 	for i := range output.HostSuggestedNetworks {
-		startIPStr := output.HostSuggestedNetworks[i].GuestIpStart
-		endIPStr := output.HostSuggestedNetworks[i].GuestIpEnd
-		startIP, _ := netutils.NewIPV4Addr(startIPStr)
-		endIP, _ := netutils.NewIPV4Addr(endIPStr)
-		excludedIRs = append(excludedIRs, netutils.NewIPV4AddrRange(startIP, endIP))
+		startipStr := output.HostSuggestedNetworks[i].GuestIpStart
+		endipStr := output.HostSuggestedNetworks[i].GuestIpEnd
+		excludeNets = append(excludeNets, &SNetwork{GuestIpEnd: endipStr, GuestIpStart: startipStr})
 	}
-	if len(noNetVMIPs) > 0 {
-		sConfs := scm.suggestVMNetwors(noNetVMIPs, excludedIRs)
-		confs := make([]api.CANetConf, len(sConfs))
+	// sort excludeNets via their GuestIpStart
+	sort.Slice(excludeNets, func(i, j int) bool {
+		return excludeNets[i].GuestIpStart < excludeNets[j].GuestIpStart
+	})
+
+	svNets := make([]netutils.IPV4AddrRange, 0, 5)
+	var vmi, neti int
+
+	lastEndIp := netutils.IPV4Addr(0)
+Loop:
+	for neti = 0; neti < len(excludeNets); {
+		if vmi >= len(vms) {
+			break
+		}
+		startIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpStart)
+		endIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpEnd)
+		switch {
+		case vms[vmi].IP > endIp:
+			lastEndIp = endIp
+			neti++
+		case vms[vmi].IP >= startIp:
+			for vms[vmi].IP <= endIp {
+				id := vms[vmi].FakeID
+				if _, ok := guestMap[id]; !ok {
+					guestMap[id] = &api.CAGuestNet{
+						Name:   vms[vmi].Name,
+						IPNets: []api.CAIPNet{},
+					}
+				}
+				guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+					IP:              vms[vmi].IP.String(),
+					SuitableNetwork: excludeNets[neti].Id,
+				})
+				vmi += 1
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+			lastEndIp = endIp
+			neti++
+		default:
+			for vms[vmi].IP < startIp {
+				suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+				if suggestStartIp <= lastEndIp {
+					suggestStartIp = lastEndIp + 1
+				}
+				if suggestStartIp >= startIp {
+					break
+				}
+				suggestEndIp := suggestStartIp.NetAddr(24) + 255
+				if suggestEndIp >= startIp {
+					suggestEndIp = startIp - 1
+				}
+				svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+				for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+					id := vms[vmi].FakeID
+					if _, ok := guestMap[id]; !ok {
+						guestMap[id] = &api.CAGuestNet{
+							Name:   vms[vmi].Name,
+							IPNets: []api.CAIPNet{},
+						}
+					}
+					guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+						IP: vms[vmi].IP.String(),
+					})
+					vmi++
+				}
+				if vmi == len(vms) {
+					break Loop
+				}
+			}
+		}
+	}
+
+	for vmi < len(vms) {
+		suggestStartIp := vms[vmi].IP.NetAddr(24) + 1
+		if suggestStartIp <= lastEndIp {
+			suggestStartIp = lastEndIp + 1
+		}
+		suggestEndIp := suggestStartIp.NetAddr(24) + 255
+		svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp))
+		for vmi < len(vms) && vms[vmi].IP <= suggestEndIp {
+			id := vms[vmi].FakeID
+			if _, ok := guestMap[id]; !ok {
+				guestMap[id] = &api.CAGuestNet{
+					Name:   vms[vmi].Name,
+					IPNets: []api.CAIPNet{},
+				}
+			}
+			guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{
+				IP: vms[vmi].IP.String(),
+			})
+			vmi++
+		}
+	}
+
+	for _, guest := range guestMap {
+		output.Guests = append(output.Guests, *guest)
+	}
+
+	if len(svNets) > 0 {
+		confs := make([]api.CANetConf, len(svNets))
 		for i := range confs {
-			confs[i].CASimpleNetConf = sConfs[i]
+			confs[i].CASimpleNetConf = api.CASimpleNetConf{
+				GuestIpStart: svNets[i].StartIp().String(),
+				GuestIpEnd:   svNets[i].EndIp().String(),
+				GuestIpMask:  24,
+				GuestGateway: (svNets[i].StartIp().NetAddr(24) + 1).String(),
+			}
 			confs[i].Name = fmt.Sprintf("%s-guest-network-%d", input.Name, i+1)
 		}
 		output.GuestSuggestedNetworks = confs
@@ -650,15 +734,19 @@ func (manager *SCloudaccountManager) fetchEsxiZoneIds() ([]string, error) {
 // The suggested network mask is 24 and the gateway is x.x.x.1.
 // The suggests network is the smallest network segment that meets the above conditions.
 func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr) []api.CASimpleNetConf {
+	if len(ips) == 0 {
+		return nil
+	}
 	sort.Slice(ips, func(i, j int) bool {
 		return ips[i] < ips[j]
 	})
 	var (
-		mask       int8 = 24
-		consequent []netutils.IPV4Addr
-		ret        []api.CASimpleNetConf
+		mask        int8 = 24
+		lastnetAddr netutils.IPV4Addr
+		consequent  []netutils.IPV4Addr
+		ret         []api.CASimpleNetConf
 	)
-	lastnetAddr, _ := netutils.NewIPV4Addr("0.0.0.0")
+	lastnetAddr = ips[0].NetAddr(mask)
 	consequent = []netutils.IPV4Addr{ips[0]}
 	for i := 1; i < len(ips); i++ {
 		ip := ips[i]
@@ -819,7 +907,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		input.AutoCreateProject = &forceAutoCreateProject
 	}
 
-	if len(input.ProjectId) > 0 {
+	if len(input.Project) > 0 {
 		var proj *db.STenant
 		proj, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
 		if err != nil {

pkg/compute/models/cloudaccounts_test.go

@@ -71,6 +71,21 @@ func TestSCloudaccount_suggestHostNetworks(t *testing.T) {
 				},
 			},
 		},
+		{
+			[]string{
+				"10.155.50.103",
+				"10.155.50.101",
+				"10.155.50.102",
+			},
+			[]api.CASimpleNetConf{
+				{
+					GuestIpStart: "10.155.50.101",
+					GuestIpEnd:   "10.155.50.103",
+					GuestIpMask:  24,
+					GuestGateway: "10.155.50.1",
+				},
+			},
+		},
 	}
 	for _, c := range cases {
 		ins := make([]netutils.IPV4Addr, len(c.in))

pkg/compute/models/cloudproviders.go

@@ -1394,8 +1394,7 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us
 	params := jsonutils.NewDict()
 	task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
-		log.Errorf("%s", err)
-		return err
+		return errors.Wrapf(err, "NewTask")
 	}
 	self.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartCloudproviderDeleteTask")
 	task.ScheduleRun(nil)

pkg/compute/models/cloudsync.go

@@ -1229,7 +1229,7 @@ func SyncCloudProject(userCred mcclient.TokenCredential, model db.IVirtualModel,
 		extProject, err := ExternalProjectManager.GetProject(extProjectId, managerId)
 		if err != nil {
 			log.Errorf("sync project for %s %s error: %v", model.Keyword(), model.GetName(), err)
-		} else {
+		} else if len(extProject.ProjectId) > 0 {
 			newOwnerId = extProject.GetOwnerId()
 		}
 	}

pkg/compute/models/dbinstance_accounts.go

@@ -730,3 +730,21 @@ func (manager *SDBInstanceAccountManager) ListItemExportKeys(ctx context.Context
 	}
 	return q, nil
 }
+
+func (manager *SDBInstanceAccountManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	accounts := []SDBInstanceAccount{}
+	err := db.FetchModelObjects(manager, q, &accounts)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range accounts {
+		err = accounts[i].Purge(context.Background(), nil)
+		if err != nil {
+			return errors.Wrapf(err, "purge %s", accounts[i].Id)
+		}
+	}
+	log.Debugf("SDBInstanceAccountManager cleaned %d dirty data.", len(accounts))
+	return nil
+}

pkg/compute/models/dbinstance_databases.go

@@ -486,3 +486,21 @@ func (manager *SDBInstanceDatabaseManager) ListItemExportKeys(ctx context.Contex
 	}
 	return q, nil
 }
+
+func (manager *SDBInstanceDatabaseManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	databases := []SDBInstanceDatabase{}
+	err := db.FetchModelObjects(manager, q, &databases)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range databases {
+		err = databases[i].Purge(context.Background(), nil)
+		if err != nil {
+			return errors.Wrapf(err, "purge %s", databases[i].Id)
+		}
+	}
+	log.Debugf("SDBInstanceDatabaseManager cleaned %d dirty data.", len(databases))
+	return nil
+}

pkg/compute/models/dbinstancenetworks.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/netutils"
@@ -81,6 +82,24 @@ func (self *SDBInstanceNetwork) AllowDeleteItem(ctx context.Context, userCred mc
 	return false
 }
 
+// RDS网络列表
+func (manager *SDBInstanceNetworkManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.DBInstanceNetworkListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SVirtualJointResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualJointResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SVirtualJointResourceBase.ListItemFilter")
+	}
+	q, err = manager.SDBInstanceResourceBaseManager.ListItemFilter(ctx, q, userCred, query.DBInstanceFilterListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SDBInstanceResourceBaseManager.ListItemFilter")
+	}
+	return q, nil
+}
+
 func (self *SDBInstanceNetwork) GetNetwork() (*SNetwork, error) {
 	network, err := NetworkManager.FetchById(self.NetworkId)
 	if err != nil {
@@ -229,3 +248,23 @@ func (manager *SDBInstanceNetworkManager) newFromCloudDBNetwork(ctx context.Cont
 	}
 	return nil
 }
+
+func (manager *SDBInstanceNetworkManager) InitializeData() error {
+	sq := DBInstanceManager.Query("id")
+	q := manager.Query().NotIn("dbinstance_id", sq.SubQuery())
+	networks := []SDBInstanceNetwork{}
+	err := db.FetchModelObjects(manager, q, &networks)
+	if err != nil {
+		return errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	for i := range networks {
+		_, err := db.Update(&networks[i], func() error {
+			return networks[i].MarkDelete()
+		})
+		if err != nil {
+			return errors.Wrapf(err, "db.Update.MarkDelete")
+		}
+	}
+	log.Debugf("SDBInstanceNetworkManager cleaned %d dirty data.", len(networks))
+	return nil
+}

pkg/compute/models/dbinstances.go

@@ -1299,6 +1299,13 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("dbInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SDBInstance, 0)
 	commondb := make([]SDBInstance, 0)
 	commonext := make([]cloudprovider.ICloudDBInstance, 0)
@@ -1344,17 +1351,7 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 }
 
 func (self *SDBInstance) syncRemoveCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential) error {
-	lockman.LockObject(ctx, self)
-	defer lockman.ReleaseObject(ctx, self)
-
-	self.SetDisableDelete(userCred, false)
-
-	err := self.ValidateDeleteCondition(ctx)
-	if err != nil { // cannot delete
-		self.SetStatus(userCred, api.VPC_STATUS_UNKNOWN, "sync to delete")
-		return errors.Wrap(err, "ValidateDeleteCondition")
-	}
-	return self.RealDelete(ctx, userCred)
+	return self.Purge(ctx, userCred)
 }
 
 func (self *SDBInstance) ValidateDeleteCondition(ctx context.Context) error {

pkg/compute/models/disks.go

@@ -1079,15 +1079,15 @@ func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) er
 		}
 		host := storage.GetMasterHost()
 		if host == nil {
-			return httperrors.NewBadRequestError("storage of disk no valid host")
+			return httperrors.NewBadRequestError("storage of disk %s no valid host", self.Id)
 		}
 	}
 	cnt, err := self.GetGuestDiskCount()
 	if err != nil {
-		return httperrors.NewInternalServerError("GetGuestDiskCount fail %s", err)
+		return httperrors.NewInternalServerError("GetGuestDiskCount for disk %s fail %s", self.Id, err)
 	}
 	if cnt > 0 {
-		return httperrors.NewNotEmptyError("Virtual disk used by virtual servers")
+		return httperrors.NewNotEmptyError("Virtual disk %s(%s) used by virtual servers", self.Name, self.Id)
 	}
 	if !isPurge && self.IsValidPrePaid() {
 		return httperrors.NewForbiddenError("not allow to delete prepaid disk in valid status")
@@ -2563,13 +2563,18 @@ func (self *SDisk) PerformChangeOwner(ctx context.Context, userCred mcclient.Tok
 	}
 	for i := range snapshots {
 		snapshot := snapshots[i]
-		lockman.LockObject(ctx, &snapshot)
-		_, err := snapshot.PerformChangeOwner(ctx, userCred, query, input)
+		err := func() error {
+			lockman.LockObject(ctx, &snapshot)
+			defer lockman.ReleaseObject(ctx, &snapshot)
+			_, err := snapshot.PerformChangeOwner(ctx, userCred, query, input)
+			if err != nil {
+				return err
+			}
+			return nil
+		}()
 		if err != nil {
-			lockman.ReleaseObject(ctx, &snapshot)
 			return nil, errors.Wrapf(err, "fail to change owner of this disk(%s)'s snapshot %s", self.Id, snapshot.Id)
 		}
-		lockman.ReleaseObject(ctx, &snapshot)
 	}
 	return nil, nil
 }

pkg/compute/models/dnsrecords.go

@@ -337,11 +337,26 @@ func (man *SDnsRecordManager) ValidateCreateData(
 	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
-	_, err := man.validateModelData(ctx, data, true)
+	var err error
+
+	input := apis.AdminSharableVirtualResourceBaseCreateInput{}
+	err = data.Unmarshal(&input)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal AdminSharableVirtualResourceBaseCreateInput")
+	}
+
+	input, err = man.SAdminSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input)
+	if err != nil {
+		return nil, errors.Wrap(err, "SAdminSharableVirtualResourceBaseManager.ValidateCreateData")
+	}
+
+	data.Update(jsonutils.Marshal(input))
+
+	_, err = man.validateModelData(ctx, data, true)
 	if err != nil {
 		return nil, err
 	}
-	return man.SAdminSharableVirtualResourceBaseManager.ValidateCreateData(man, data)
+	return man.SAdminSharableVirtualResourceBaseManager.ValidateRecordsData(man, data)
 }
 
 func (man *SDnsRecordManager) QueryDns(projectId, name string) *SDnsRecord {

pkg/compute/models/elasticcache_instances.go

@@ -452,6 +452,13 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("ElasticCacheInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SElasticcache, 0)
 	commondb := make([]SElasticcache, 0)
 	commonext := make([]cloudprovider.ICloudElasticcache, 0)
@@ -502,6 +509,8 @@ func (self *SElasticcache) syncRemoveCloudElasticcache(ctx context.Context, user
 
 	self.SetDisableDelete(userCred, false)
 
+	self.DeleteSubResources(ctx, userCred)
+
 	err := self.ValidateDeleteCondition(ctx)
 	if err != nil {
 		self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_ERROR, "sync to delete")

pkg/compute/models/elasticips.go

@@ -29,6 +29,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
@@ -426,8 +427,11 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m
 			case api.EIP_ASSOCIATE_TYPE_SERVER:
 				sq := HostManager.Query().SubQuery()
 				return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("host_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), self.ManagerId))
-			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY, api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
+			case api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
 				return q.Equals("manager_id", self.ManagerId)
+			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY:
+				sq := VpcManager.Query("id").Equals("manager_id", self.ManagerId)
+				return q.In("vpc_id", sq.SubQuery())
 			}
 			return q
 		})
@@ -795,51 +799,20 @@ func (manager *SElasticipManager) getEipByExtEip(ctx context.Context, userCred m
 	return manager.newFromCloudEip(ctx, userCred, extEip, provider, region, syncOwnerId)
 }
 
-func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.SElasticipCreateInput) (*jsonutils.JSONDict, error) {
-	var (
-		region   *SCloudregion
-		provider *SCloudprovider
-		err      error
-	)
-	for _, cloudregion := range []string{input.Cloudregion, input.Region, input.RegionId} {
-		if len(cloudregion) > 0 {
-			input.Cloudregion = cloudregion
-			break
-		}
+func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.SElasticipCreateInput) (api.SElasticipCreateInput, error) {
+	if input.CloudregionId == "" {
+		input.CloudregionId = api.DEFAULT_REGION_ID
 	}
-	if input.Cloudregion == "" {
-		input.Cloudregion = api.DEFAULT_REGION_ID
-	}
-	if obj, err := CloudregionManager.FetchByIdOrName(nil, input.Cloudregion); err != nil {
+	obj, err := CloudregionManager.FetchByIdOrName(nil, input.CloudregionId)
+	if err != nil {
 		if err != sql.ErrNoRows {
-			return nil, httperrors.NewGeneralError(err)
-		} else {
-			return nil, httperrors.NewResourceNotFoundError("Region %s not found", input.Cloudregion)
+			return input, httperrors.NewGeneralError(err)
 		}
-	} else {
-		region = obj.(*SCloudregion)
+		return input, httperrors.NewResourceNotFoundError2("cloudregion", input.CloudregionId)
 	}
+	region := obj.(*SCloudregion)
 	input.CloudregionId = region.GetId()
 
-	for _, cloudprovider := range []string{input.Cloudprovider, input.Manager, input.ManagerId} {
-		if len(cloudprovider) > 0 {
-			input.Cloudprovider = cloudprovider
-			break
-		}
-	}
-	if input.Cloudprovider != "" {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(nil, input.Cloudprovider)
-		if err != nil {
-			if err != sql.ErrNoRows {
-				return nil, httperrors.NewGeneralError(err)
-			} else {
-				return nil, httperrors.NewResourceNotFoundError("Cloud provider %s not found", input.Cloudprovider)
-			}
-		}
-		provider = providerObj.(*SCloudprovider)
-		input.ManagerId = provider.Id
-	}
-
 	// publicIp cannot be created standalone
 	input.Mode = api.EIP_MODE_STANDALONE_EIP
 
@@ -848,15 +821,30 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 	}
 
 	if !utils.IsInStringArray(input.ChargeType, []string{api.EIP_CHARGE_TYPE_BY_BANDWIDTH, api.EIP_CHARGE_TYPE_BY_TRAFFIC}) {
-		return nil, httperrors.NewInputParameterError("charge type %s not supported", input.ChargeType)
+		return input, httperrors.NewInputParameterError("charge type %s not supported", input.ChargeType)
 	}
 
-	if input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput); err != nil {
-		return nil, err
+	input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
+	if err != nil {
+		return input, err
 	}
 
-	if err = region.GetDriver().ValidateCreateEipData(ctx, userCred, &input); err != nil {
-		return nil, err
+	err = region.GetDriver().ValidateCreateEipData(ctx, userCred, &input)
+	if err != nil {
+		return input, err
+	}
+
+	var provider *SCloudprovider = nil
+	if input.ManagerId != "" {
+		providerObj, err := CloudproviderManager.FetchByIdOrName(nil, input.ManagerId)
+		if err != nil {
+			if err != sql.ErrNoRows {
+				return input, httperrors.NewGeneralError(err)
+			}
+			return input, httperrors.NewResourceNotFoundError2("cloudprovider", input.ManagerId)
+		}
+		provider = providerObj.(*SCloudprovider)
+		input.ManagerId = provider.Id
 	}
 
 	//避免参数重名后还有pending.eip残留
@@ -864,10 +852,10 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr
 	quotaKeys := fetchRegionalQuotaKeys(rbacutils.ScopeProject, ownerId, region, provider)
 	eipPendingUsage.SetKeys(quotaKeys)
 	if err = quotas.CheckSetPendingQuota(ctx, userCred, eipPendingUsage); err != nil {
-		return nil, err
+		return input, err
 	}
 
-	return input.JSON(input), nil
+	return input, nil
 }
 
 func (eip *SElasticip) GetQuotaKeys() (quotas.IQuotaKeys, error) {
@@ -1246,8 +1234,8 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		chargeType = api.EIP_CHARGE_TYPE_BY_TRAFFIC
 	}
 
-	eip := SElasticip{}
-	eip.SetModelManager(manager, &eip)
+	eip := &SElasticip{}
+	eip.SetModelManager(manager, eip)
 
 	eip.Mode = api.EIP_MODE_STANDALONE_EIP
 	// do not implicitly auto dellocate EIP, should be set by user explicitly
@@ -1265,7 +1253,9 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		hostq := HostManager.Query().SubQuery()
 		wireq := WireManager.Query().SubQuery()
 		hostwireq := HostwireManager.Query().SubQuery()
+		scope := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), NetworkManager.KeywordPlural(), policy.PolicyActionList)
 		q := NetworkManager.Query()
+		q = NetworkManager.FilterByOwner(q, userCred, scope)
 		q = q.Join(wireq, sqlchemy.Equals(wireq.Field("id"), q.Field("wire_id")))
 		q = q.Join(hostwireq, sqlchemy.Equals(hostwireq.Field("wire_id"), wireq.Field("id")))
 		q = q.Join(hostq, sqlchemy.Equals(hostq.Field("id"), host.Id))
@@ -1275,17 +1265,21 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 			return nil, errors.Wrapf(err, "fetch eip networks usable in host %s(%s)",
 				host.Name, host.Id)
 		}
+		var net *SNetwork
 		for i := range nets {
-			net := &nets[i]
+			net = &nets[i]
 			cnt, err := net.GetFreeAddressCount()
 			if err != nil {
 				continue
 			}
 			if cnt > 0 {
-				eip.NetworkId = net.Id
 				break
 			}
 		}
+		if net == nil {
+			return nil, errors.Error("no usable eip network")
+		}
+		eip.NetworkId = net.Id
 	}
 
 	var err error
@@ -1294,11 +1288,12 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		return nil, errors.Wrap(err, "db.GenerateName")
 	}
 
-	err = manager.TableSpec().Insert(ctx, &eip)
+	err = manager.TableSpec().Insert(ctx, eip)
 	if err != nil {
 		log.Errorf("create EIP record fail %s", err)
 		return nil, err
 	}
+	db.OpsLog.LogEvent(eip, db.ACT_CREATE, eip.GetShortDesc(ctx), userCred)
 
 	eipPendingUsage := &SRegionQuota{Eip: 1}
 	keys := fetchRegionalQuotaKeys(
@@ -1310,7 +1305,7 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 	eipPendingUsage.SetKeys(keys)
 	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage, true)
 
-	return &eip, nil
+	return eip, nil
 }
 
 func (eip *SElasticip) AllocateAndAssociateVM(ctx context.Context, userCred mcclient.TokenCredential, vm *SGuest, parentTaskId string) error {

pkg/compute/models/external_projects.go

@@ -26,11 +26,15 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -198,6 +202,25 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 		self.Name = ext.GetName()
 		self.IsEmulated = ext.IsEmulated()
 		self.Status = ext.GetStatus()
+		if self.DomainId == account.DomainId && account.AutoCreateProject && options.Options.EnableAutoRenameProject {
+			tenant, err := db.TenantCacheManager.FetchTenantById(ctx, self.ProjectId)
+			if err != nil {
+				return errors.Wrapf(err, "TenantCacheManager.FetchTenantById(%s)", self.ProjectId)
+			}
+			if tenant.Name != self.Name {
+				s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+				params := map[string]string{"name": self.Name}
+				_, err := modules.Projects.Update(s, tenant.Id, jsonutils.Marshal(params))
+				if err != nil {
+					return errors.Wrapf(err, "update project name from %s -> %s", tenant.Name, self.Name)
+				}
+				_, err = db.Update(tenant, func() error {
+					tenant.Name = self.Name
+					return nil
+				})
+				return err
+			}
+		}
 		if self.DomainId != account.DomainId {
 			self.ProjectId = account.ProjectId
 			self.DomainId = account.DomainId
@@ -205,8 +228,7 @@ func (self *SExternalProject) SyncWithCloudProject(ctx context.Context, userCred
 		return nil
 	})
 	if err != nil {
-		log.Errorf("SyncWithCloudProject fail %s", err)
-		return err
+		return errors.Wrapf(err, "db.UpdateWithLock")
 	}
 	db.OpsLog.LogSyncUpdate(self, diff, userCred)
 	return nil

pkg/compute/models/guest_actions.go

@@ -89,7 +89,7 @@ func (self *SGuest) PreCheckPerformAction(
 ) error {
 	if self.Hypervisor == api.HYPERVISOR_KVM {
 		host := self.GetHost()
-		if (host.HostStatus == api.HOST_OFFLINE || !host.Enabled.Bool()) &&
+		if host != nil && (host.HostStatus == api.HOST_OFFLINE || !host.Enabled.Bool()) &&
 			utils.IsInStringArray(action,
 				[]string{
 					"start", "restart", "stop", "reset", "rebuild-root",
@@ -3047,6 +3047,13 @@ func (self *SGuest) PerformSetQemuParams(ctx context.Context, userCred mcclient.
 			return nil, err
 		}
 	}
+	usbKbd, err := data.GetString("disable_usb_kbd")
+	if err == nil {
+		err = self.SetMetadata(ctx, "disable_usb_kbd", usbKbd, userCred)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return nil, nil
 }
 
@@ -3628,9 +3635,11 @@ func (self *SGuest) SaveRenewInfo(
 	guestdisks := self.GetDisks()
 	for i := 0; i < len(guestdisks); i += 1 {
 		disk := guestdisks[i].GetDisk()
-		err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
-		if err != nil {
-			return err
+		if disk.AutoDelete {
+			err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil

pkg/compute/models/guestdisks.go

@@ -187,7 +187,7 @@ func (self *SGuestdisk) DoSave(ctx context.Context, driver string, cache string,
 func (self *SGuestdisk) GetDisk() *SDisk {
 	disk, err := DiskManager.FetchById(self.DiskId)
 	if err != nil {
-		log.Errorf("GetDisk fail: %s", err)
+		log.Errorf("GetDisk %s fail: %s", self.DiskId, err)
 		return nil
 	}
 	return disk.(*SDisk)
@@ -293,6 +293,10 @@ func (self *SGuestdisk) GetDetailedInfo() api.GuestDiskInfo {
 
 func (self *SGuestdisk) GetDetailedString() string {
 	disk := self.GetDisk()
+	if disk == nil {
+		return ""
+	}
+
 	var fs string
 	if len(disk.GetTemplateId()) > 0 {
 		fs = "root"

pkg/compute/models/guestdrivers.go

@@ -142,6 +142,8 @@ type IGuestDriver interface {
 	IsRebuildRootSupportChangeImage() bool
 	IsRebuildRootSupportChangeUEFI() bool
 
+	IsSupportdDcryptPasswordFromSecretKey() bool
+
 	RequestDeleteDetachedDisk(ctx context.Context, disk *SDisk, task taskman.ITask, isPurge bool) error
 	StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error
 	StartGuestAttachDiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error

pkg/compute/models/guestnetworks.go

@@ -644,6 +644,8 @@ func totalGuestNicCount(
 	q = q.Join(guests, sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id")))
 	q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id")))
 
+	q = q.Filter(sqlchemy.IsFalse(guests.Field("pending_deleted")))
+
 	q = CloudProviderFilter(q, hosts.Field("manager_id"), providers, brands, cloudEnv)
 	q = RangeObjectsFilter(q, rangeObjs, nil, hosts.Field("zone_id"), hosts.Field("manager_id"), hosts.Field("id"), nil)
 

pkg/compute/models/guests.go

@@ -1149,6 +1149,7 @@ func (manager *SGuestManager) validateCreateData(
 		if len(imgProperties) == 0 {
 			imgProperties = map[string]string{"os_type": "Linux"}
 		}
+		input.DisableUsbKbd = imgProperties[imageapi.IMAGE_DISABLE_USB_KBD] == "true"
 
 		osType := input.OsType
 		osProf, err = osprofile.GetOSProfileFromImageProperties(imgProperties, hypervisor)
@@ -1707,6 +1708,9 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 	if osProfileJson != nil {
 		guest.setOSProfile(ctx, userCred, osProfileJson)
 	}
+	if jsonutils.QueryBoolean(data, imageapi.IMAGE_DISABLE_USB_KBD, false) {
+		guest.SetMetadata(ctx, imageapi.IMAGE_DISABLE_USB_KBD, "true", userCred)
+	}
 
 	userData, _ := data.GetString("user_data")
 	if len(userData) > 0 {
@@ -2012,8 +2016,10 @@ func (self *SGuest) getNetworksDetails() string {
 func (self *SGuest) getDisksDetails() string {
 	var buf bytes.Buffer
 	for _, disk := range self.GetDisks() {
-		buf.WriteString(disk.GetDetailedString())
-		buf.WriteString("\n")
+		if details := disk.GetDetailedString(); len(details) > 0 {
+			buf.WriteString(details)
+			buf.WriteString("\n")
+		}
 	}
 	return buf.String()
 }
@@ -2769,7 +2775,7 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host
 	if vnet == nil {
 		if vnic.InClassicNetwork() {
 			region := host.GetRegion()
-			cloudprovider := region.GetCloudprovider()
+			cloudprovider := host.GetCloudprovider()
 			vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, cloudprovider, region)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
@@ -2975,8 +2981,8 @@ func (self *SGuest) attach2Disk(ctx context.Context, disk *SDisk, userCred mccli
 	guestdisk.DiskId = disk.Id
 	guestdisk.GuestId = self.Id
 
-	defer lockman.ReleaseObject(ctx, self)
 	lockman.LockObject(ctx, self)
+	defer lockman.ReleaseObject(ctx, self)
 
 	guestdisk.Index = self.getDiskIndex()
 	err = guestdisk.DoSave(ctx, driver, cache, mountpoint)
@@ -3627,7 +3633,7 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To
 		disk := guestdisk.GetDisk()
 		err := guestdisk.Detach(ctx, userCred)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "guestdisk.Detach guest_id: %s disk_id: %s", guestdisk.GuestId, guestdisk.DiskId)
 		}
 
 		if disk != nil {
@@ -4230,6 +4236,14 @@ func (self *SGuest) GetShortDesc(ctx context.Context) *jsonutils.JSONDict {
 		billingInfo.SCloudProviderInfo = host.getCloudProviderInfo()
 	}
 
+	if len(self.BackupHostId) > 0 {
+		backupHost := HostManager.FetchHostById(self.BackupHostId)
+		if backupHost != nil {
+			desc.Set("backup_host", jsonutils.NewString(backupHost.Name))
+			desc.Set("backup_host_id", jsonutils.NewString(backupHost.Id))
+		}
+	}
+
 	if priceKey := self.GetMetadata("ext:price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}
@@ -4253,40 +4267,49 @@ func (self *SGuest) saveOsType(userCred mcclient.TokenCredential, osType string)
 	return err
 }
 
+type sDeployInfo struct {
+	Os       string
+	Account  string
+	Key      string
+	Distro   string
+	Version  string
+	Arch     string
+	Language string
+}
+
 func (self *SGuest) SaveDeployInfo(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) {
+	deployInfo := sDeployInfo{}
+	data.Unmarshal(&deployInfo)
 	info := make(map[string]interface{})
-	if data.Contains("os") {
-		osName, _ := data.GetString("os")
-		self.saveOsType(userCred, osName)
-		info["os_name"] = osName
+	if len(deployInfo.Os) > 0 {
+		self.saveOsType(userCred, deployInfo.Os)
+		info["os_name"] = deployInfo.Os
 	}
-	if data.Contains("account") {
-		account, _ := data.GetString("account")
-		info["login_account"] = account
-		if data.Contains("key") {
-			key, _ := data.GetString("key")
-			info["login_key"] = key
+	driver := self.GetDriver()
+	if len(deployInfo.Account) > 0 {
+		info["login_account"] = deployInfo.Account
+		if len(deployInfo.Key) > 0 {
+			info["login_key"] = deployInfo.Key
+			if len(self.KeypairId) > 0 && !driver.IsSupportdDcryptPasswordFromSecretKey() { // Tencent Cloud does not support simultaneous setting of secret keys and passwords
+				info["login_key"], _ = seclib2.EncryptBase64(self.GetKeypairPublicKey(), "")
+			}
 			info["login_key_timestamp"] = timeutils.UtcNow()
 		} else {
 			info["login_key"] = "none"
 			info["login_key_timestamp"] = "none"
 		}
 	}
-	if data.Contains("distro") {
-		dist, _ := data.GetString("distro")
-		info["os_distribution"] = dist
+	if len(deployInfo.Distro) > 0 {
+		info["os_distribution"] = deployInfo.Distro
 	}
-	if data.Contains("version") {
-		ver, _ := data.GetString("version")
-		info["os_version"] = ver
+	if len(deployInfo.Version) > 0 {
+		info["os_version"] = deployInfo.Version
 	}
-	if data.Contains("arch") {
-		arch, _ := data.GetString("arch")
-		info["os_arch"] = arch
+	if len(deployInfo.Arch) > 0 {
+		info["os_arch"] = deployInfo.Arch
 	}
-	if data.Contains("language") {
-		lang, _ := data.GetString("language")
-		info["os_language"] = lang
+	if len(deployInfo.Language) > 0 {
+		info["os_language"] = deployInfo.Language
 	}
 	self.SetAllMetadata(ctx, info, userCred)
 	self.saveOldPassword(ctx, userCred)

pkg/compute/models/hosts.go

@@ -674,7 +674,7 @@ func (self *SHost) RealDelete(ctx context.Context, userCred mcclient.TokenCreden
 		if storage != nil && storage.IsLocal() {
 			cnt, err := storage.GetDiskCount()
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "GetDiskCount")
 			}
 			if cnt > 0 {
 				return httperrors.NewNotEmptyError("Inconsistent: local storage is not empty???")
@@ -1418,6 +1418,17 @@ func (self *SHost) GetGuests() []SGuest {
 	return guests
 }
 
+func (self *SHost) GetKvmGuests() []SGuest {
+	q := GuestManager.Query().Equals("host_id", self.Id).Equals("hypervisor", api.HYPERVISOR_KVM)
+	guests := make([]SGuest, 0)
+	err := db.FetchModelObjects(GuestManager, q, &guests)
+	if err != nil {
+		log.Errorf("GetGuests %s", err)
+		return nil
+	}
+	return guests
+}
+
 func (self *SHost) GetGuestsMasterOnThisHost() []SGuest {
 	q := self.GetGuestsQuery().IsNotEmpty("backup_host_id")
 	guests := make([]SGuest, 0)
@@ -2508,11 +2519,7 @@ func (self *SHost) GetIHostAndProvider() (cloudprovider.ICloudHost, cloudprovide
 	}
 	ihost, err := iregion.GetIHostById(self.ExternalId)
 	if err != nil {
-		if err == cloudprovider.ErrNotFound {
-			return nil, nil, cloudprovider.ErrNotFound
-		}
-		log.Errorf("fail to find ihost by id %s %s", self.ExternalId, err)
-		return nil, nil, fmt.Errorf("fail to find ihost by id %s", err)
+		return nil, nil, errors.Wrapf(err, "iregion.GetIHostById(%s)", self.ExternalId)
 	}
 	return ihost, provider, nil
 }
@@ -3519,6 +3526,9 @@ func (self *SHost) PerformOffline(ctx context.Context, userCred mcclient.TokenCr
 			if jsonutils.QueryBoolean(data, "update_health_status", false) {
 				self.EnableHealthCheck = false
 			}
+			// Note: update host status to unknown on host offline
+			// we did not have host status after host offline
+			self.Status = api.BAREMETAL_UNKNOWN
 			return nil
 		})
 		if err != nil {
@@ -4612,7 +4622,15 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 						enables = append(enables, extNics[i])
 					}
 				} else {
-					// do nothing, in sync
+					// in sync, sync interface and bridge
+					hw := host.getHostwireOfIdAndMac(netIfs[i].WireId, netIfs[i].Mac)
+					if hw != nil && (hw.Bridge != extNics[i].GetBridge() || hw.Interface != extNics[i].GetDevice()) {
+						db.Update(hw, func() error {
+							hw.Interface = extNics[i].GetDevice()
+							hw.Bridge = extNics[i].GetBridge()
+							return nil
+						})
+					}
 				}
 			} else {
 				reserveIp := false
@@ -4663,7 +4681,7 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 		// always try reserved pool
 		extNic := adds[i].netif
 		err = host.addNetif(ctx, userCred, extNic.GetMac(), "", extNic.GetIpAddr(), 0, extNic.GetNicType(), extNic.GetIndex(),
-			extNic.IsLinkUp(), int16(extNic.GetMtu()), false, "", "", true, true)
+			extNic.IsLinkUp(), int16(extNic.GetMtu()), false, extNic.GetDevice(), extNic.GetBridge(), true, true)
 		if err != nil {
 			result.AddError(err)
 		} else {
@@ -4813,10 +4831,12 @@ func (manager *SHostManager) PingDetectionTask(ctx context.Context, userCred mcc
 		var host = new(SHost)
 		q.Row2Struct(rows, host)
 		host.SetModelManager(manager, host)
-		lockman.LockObject(ctx, host)
-		host.PerformOffline(ctx, userCred, nil, data)
-		host.MarkGuestUnknown(userCred)
-		lockman.ReleaseObject(ctx, host)
+		func() {
+			lockman.LockObject(ctx, host)
+			defer lockman.ReleaseObject(ctx, host)
+			host.PerformOffline(ctx, userCred, nil, data)
+			host.MarkGuestUnknown(userCred)
+		}()
 	}
 }
 
@@ -4896,7 +4916,7 @@ func (host *SHost) PerformHostMaintenance(ctx context.Context, userCred mcclient
 		preferHostId = host.Id
 	}
 
-	guests := host.GetGuests()
+	guests := host.GetKvmGuests()
 	for i := 0; i < len(guests); i++ {
 		lockman.LockObject(ctx, &guests[i])
 		defer lockman.ReleaseObject(ctx, &guests[i])
@@ -5428,3 +5448,15 @@ func (manager *SHostManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SHostManager) FetchHostByExtId(extid string) *SHost {
+	host := SHost{}
+	host.SetModelManager(manager, &host)
+	err := manager.Query().Equals("external_id", extid).First(&host)
+	if err != nil {
+		log.Errorf("fetchHostByExtId fail %s", err)
+		return nil
+	} else {
+		return &host
+	}
+}

pkg/compute/models/hostschedtags.go

@@ -44,7 +44,6 @@ func init() {
 				"schedtaghost",
 				"schedtaghosts",
 				HostManager,
-				SchedtagManager,
 			),
 		}
 		HostschedtagManager.SetVirtualObject(HostschedtagManager)
@@ -57,7 +56,7 @@ type SHostschedtag struct {
 	HostId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False)
 }
 
-func (manager *SHostschedtagManager) GetSlaveFieldName() string {
+func (manager *SHostschedtagManager) GetMasterFieldName() string {
 	return "host_id"
 }
 

pkg/compute/models/initdb.go

@@ -52,6 +52,7 @@ func InitDB() error {
 		LoadbalancerListenerRuleManager,
 		LoadbalancerBackendGroupManager,
 		LoadbalancerBackendManager,
+		AwsCachedLbbgManager,
 		LoadbalancerClusterManager,
 		SchedtagManager,
 		DynamicschedtagManager,
@@ -60,6 +61,12 @@ func InitDB() error {
 
 		ScheduledTaskActivityManager,
 		ExternalProjectManager,
+		CachedimageManager,
+		StoragecachedimageManager,
+		NetworkinterfacenetworkManager,
+		DBInstanceNetworkManager,
+		DBInstanceAccountManager,
+		DBInstanceDatabaseManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/loadbalancerawscachedlbb.go

@@ -159,16 +159,14 @@ func (man *SAwsCachedLbManager) SyncLoadbalancerBackends(ctx context.Context, us
 		if err != nil {
 			syncResult.UpdateError(err)
 		} else {
-			syncMetadata(ctx, userCred, &commondb[i], commonext[i])
 			syncResult.Update()
 		}
 	}
 	for i := 0; i < len(added); i++ {
-		local, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
+		_, err := man.newFromCloudLoadbalancerBackend(ctx, userCred, loadbalancerBackendgroup, added[i], syncOwnerId)
 		if err != nil {
 			syncResult.AddError(err)
 		} else {
-			syncMetadata(ctx, userCred, local, added[i])
 			syncResult.Add()
 		}
 	}