Merge pull request #7771 from swordqiu/automated-cherry-pick-of-#7769-upstream-release-3.3

Automated cherry pick of #7769: fix: network address list respect privilige of user

cmd/climc/shell/compute/networks.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -360,15 +361,18 @@ func init() {
 
 	type NetworkAddressOptions struct {
 		NETWORK string `help:"id or name of network to query"`
+
+		api.GetNetworkAddressesInput
 	}
 	R(&NetworkAddressOptions{}, "network-addresses", "Query used addresses of network", func(s *mcclient.ClientSession, args *NetworkAddressOptions) error {
-		result, err := modules.Networks.GetSpecific(s, args.NETWORK, "addresses", nil)
+		result, err := modules.Networks.GetSpecific(s, args.NETWORK, "addresses", jsonutils.Marshal(args.GetNetworkAddressesInput))
 		if err != nil {
 			return err
 		}
-		addrList, err := result.GetArray("addresses")
-		if err != nil {
-			return err
+		addrList, _ := result.GetArray("addresses")
+		if addrList == nil {
+			fmt.Println("no result")
+			return nil
 		}
 		listResult := modulebase.ListResult{Data: addrList}
 		printList(&listResult, nil)

cmd/raidcli/main.go

@@ -0,0 +1,167 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	_ "yunion.io/x/onecloud/cmd/raidcli/shell"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/ssh"
+)
+
+type BaseOptions struct {
+	Debug      bool   `help:"debug mode"`
+	Help       bool   `help:"Show help"`
+	Host       string `help:"SSH Host IP" default:"$RAID_HOST" metavar:"RAID_HOST"`
+	Username   string `help:"Username, usually root" default:"$RAID_USERNAME" metavar:"RAID_USERNAME"`
+	Password   string `help:"Password" default:"$RAID_PASSWORD" metavar:"RAID_PASSWORD"`
+	Driver     string `help:"Password" default:"$RAID_DRIVER" metavar:"RAID_DRIVER" choices:"MegaRaid|HPSARaid|Mpt2SAS|MarvelRaid"`
+	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
+}
+
+var (
+	options = &BaseOptions{}
+)
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(options,
+		"raidcli",
+		"Command-line interface to test RAID drivers.",
+		`See "raidcli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient() (raid.IRaidDriver, error) {
+	if len(options.Host) == 0 {
+		return nil, fmt.Errorf("Missing host")
+	}
+
+	if len(options.Username) == 0 {
+		return nil, fmt.Errorf("Missing username")
+	}
+
+	if len(options.Password) == 0 {
+		return nil, fmt.Errorf("Missing password")
+	}
+
+	if len(options.Driver) == 0 {
+		return nil, fmt.Errorf("Missing driver")
+	}
+
+	if options.Debug {
+		raid.Debug = true
+	}
+
+	sshClient, err := ssh.NewClient(
+		options.Host,
+		22,
+		options.Username,
+		options.Password,
+		"",
+	)
+	if err != nil {
+		return nil, fmt.Errorf("ssh client init fail: %s", err)
+	}
+
+	drv := drivers.GetDriver(options.Driver, sshClient)
+	if drv == nil {
+		return nil, fmt.Errorf("not supported driver %s", options.Driver)
+	}
+
+	err = drv.ParsePhyDevs()
+	if err != nil {
+		return nil, fmt.Errorf("parse phyical devices error %s", err)
+	}
+
+	return drv, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	// options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var client raid.IRaidDriver
+				client, e = newClient()
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(client, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/raidcli/shell/raid.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type ShowOptions struct {
+	}
+	shellutils.R(&ShowOptions{}, "show", "show raid info", func(drv raid.IRaidDriver, args *ShowOptions) error {
+		adpts := drv.GetAdapters()
+		for i := range adpts {
+			fmt.Println("Index:", i)
+			fmt.Println(jsonutils.Marshal(adpts[i].GetDevices()).PrettyString())
+		}
+		return nil
+	})
+}

cmd/redfishcli/main.go

@@ -17,12 +17,16 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
+	"strings"
 
 	"yunion.io/x/structarg"
 
 	_ "yunion.io/x/onecloud/cmd/redfishcli/shell"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/redfish"
+	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
 	_ "yunion.io/x/onecloud/pkg/util/redfish/loader"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -36,8 +40,12 @@ type BaseOptions struct {
 	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
 }
 
+var (
+	options = &BaseOptions{}
+)
+
 func getSubcommandParser() (*structarg.ArgumentParser, error) {
-	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+	parse, e := structarg.NewArgumentParser(options,
 		"redfishcli",
 		"Command-line interface to redfish API.",
 		`See "redfishcli help COMMAND" for help on a specific command.`)
@@ -62,6 +70,7 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 			return nil
 		}
 	})
+	bmcJnlp()
 	for _, v := range shellutils.CommandTable {
 		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
 		if e != nil {
@@ -71,13 +80,49 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 	return parse, nil
 }
 
+func bmcJnlp() {
+	type BmcGetOptions struct {
+		BRAND string `help:"brand of baremetal" choices:"Lenovo|Huawei|HPE|Dell|Supermicro"`
+		Save  string `help:"save to file"`
+		Debug bool   `help:"turn on debug mode"`
+	}
+	shellutils.R(&BmcGetOptions{}, "bmc-jnlp", "Get Java Console JNLP file", func(args *BmcGetOptions) error {
+		ctx := context.Background()
+		parts, err := url.Parse(options.Endpoint)
+		if err != nil {
+			return err
+		}
+		bmc := bmconsole.NewBMCConsole(parts.Hostname(), options.Username, options.Password, args.Debug)
+		var jnlp string
+		switch strings.ToLower(args.BRAND) {
+		case "hp", "hpe":
+			jnlp, err = bmc.GetIloConsoleJNLP(ctx)
+		case "dell", "dell inc.":
+			jnlp, err = bmc.GetIdracConsoleJNLP(ctx, "", "")
+		case "supermicro":
+			jnlp, err = bmc.GetSupermicroConsoleJNLP(ctx)
+		case "lenovo":
+			jnlp, err = bmc.GetLenovoConsoleJNLP(ctx)
+		}
+		if err != nil {
+			return err
+		}
+		if len(args.Save) > 0 {
+			return fileutils2.FilePutContents(args.Save, jnlp, false)
+		} else {
+			fmt.Println(jnlp)
+			return nil
+		}
+	})
+}
+
 func showErrorAndExit(e error) {
 	fmt.Fprintf(os.Stderr, "%s", e)
 	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 
-func newClient(options *BaseOptions) (redfish.IRedfishDriver, error) {
+func newClient() (redfish.IRedfishDriver, error) {
 	if len(options.Endpoint) == 0 {
 		return nil, fmt.Errorf("Missing endpoint")
 	}
@@ -104,7 +149,7 @@ func main() {
 		showErrorAndExit(e)
 	}
 	e = parser.ParseArgs(os.Args[1:], false)
-	options := parser.Options().(*BaseOptions)
+	// options := parser.Options().(*BaseOptions)
 
 	if options.Help {
 		fmt.Print(parser.HelpString())
@@ -122,9 +167,11 @@ func main() {
 			suboptions := subparser.Options()
 			if options.SUBCOMMAND == "help" {
 				e = subcmd.Invoke(suboptions)
+			} else if options.SUBCOMMAND == "bmc-jnlp" {
+				e = subcmd.Invoke(suboptions)
 			} else {
 				var client redfish.IRedfishDriver
-				client, e = newClient(options)
+				client, e = newClient()
 				if e != nil {
 					showErrorAndExit(e)
 				}

pkg/apigateway/handler/misc.go

@@ -53,6 +53,7 @@ const (
 	HOST_IPMI_USERNAME_OPTIONAL = "IPMI用户名"
 	HOST_IPMI_PASSWORD_OPTIONAL = "IPMI密码"
 	HOST_MNG_IP_ADDR_OPTIONAL   = "管理口IP地址"
+	HOST_MNG_MAC_ADDR_OPTIONAL  = "管理口MAC地址"
 )
 
 const (
@@ -63,7 +64,7 @@ const (
 var (
 	BatchHostRegisterTemplate    = []string{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}
 	BatchHostISORegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}
-	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}
+	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_MAC_ADDR_OPTIONAL, HOST_MNG_IP_ADDR_OPTIONAL}
 )
 
 func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
@@ -207,7 +208,7 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	i2 := -1
 	for i, title := range rows[0] {
 		switch title {
-		case HOST_MAC:
+		case HOST_MAC, HOST_MNG_MAC_ADDR_OPTIONAL:
 			paramKeys = append(paramKeys, "access_mac")
 		case HOST_NAME:
 			paramKeys = append(paramKeys, "name")

pkg/apis/compute/network.go

@@ -300,3 +300,13 @@ type NetworkUpdateInput struct {
 	// 分配策略
 	AllocPolicy string `json:"alloc_policy"`
 }
+
+type GetNetworkAddressesInput struct {
+	// 获取资源的范围，例如 project|domain|system
+	Scope string `json:"scope"`
+}
+
+type GetNetworkAddressesOutput struct {
+	// IP子网地址记录
+	Addresses []SNetworkAddress `json:"addresses"`
+}

pkg/baremetal/manager.go

@@ -1953,6 +1953,8 @@ func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error)
 		return bmc.GetIdracConsoleJNLP(ctx, "", "")
 	case "supermicro":
 		return bmc.GetSupermicroConsoleJNLP(ctx)
+	case "lenovo":
+		return bmc.GetLenovoConsoleJNLP(ctx)
 	}
 	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
 }

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -20,9 +20,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/pkg/utils"
@@ -75,7 +74,7 @@ func (dev *MegaRaidPhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalSto
 }
 
 func (dev *MegaRaidPhyDev) GetSize() int64 {
-	return dev.sector * int64(dev.block) / 1024 / 1024 // MB
+	return dev.sector * dev.block / 1024 / 1024 // MB
 }
 
 func (dev *MegaRaidPhyDev) parseLine(line string) bool {
@@ -124,7 +123,7 @@ func (dev *MegaRaidPhyDev) parseLine(line string) bool {
 		if err != nil {
 			log.Errorf("parse logical sector size error: %v", err)
 			dev.block = 512
-		} else {
+		} else if block > 0 {
 			dev.block = int64(block)
 		}
 	default:
@@ -189,6 +188,7 @@ type MegaRaidAdaptor struct {
 	raid         *MegaRaid
 	devs         []*MegaRaidPhyDev
 	sn           string
+	name         string
 	busNumber    string
 	deviceNumber string
 	funcNumber   string
@@ -209,6 +209,10 @@ func NewMegaRaidAdaptor(index int, raid *MegaRaid) (*MegaRaidAdaptor, error) {
 	return adapter, nil
 }
 
+func (adapter MegaRaidAdaptor) key() string {
+	return adapter.name + adapter.sn
+}
+
 func (adapter *MegaRaidAdaptor) fillInfo() error {
 	cmd := GetCommand("-AdpAllInfo", fmt.Sprintf("-a%d", adapter.index))
 	ret, err := adapter.remoteRun(cmd)
@@ -223,10 +227,12 @@ func (adapter *MegaRaidAdaptor) fillInfo() error {
 		switch key {
 		case "Serial No":
 			adapter.sn = val
+		case "Product Name":
+			adapter.name = val
 		}
 	}
-	if len(adapter.sn) == 0 {
-		return errors.New("Not found Serial No")
+	if len(adapter.key()) == 0 {
+		return errors.Error("Not found Serial No and Product Name")
 	}
 	return adapter.fillPCIInfo()
 }
@@ -267,7 +273,7 @@ func (adapter *MegaRaidAdaptor) fillPCIInfo() error {
 		}
 	}
 	if len(adapter.busNumber) == 0 || len(adapter.deviceNumber) == 0 || len(adapter.funcNumber) == 0 {
-		return errors.New("Not found bus number")
+		return errors.Error("Not found bus number")
 	}
 	pciDir := fmt.Sprintf("/sys/bus/pci/devices/0000:%s:%s.%s/", adapter.busNumber, adapter.deviceNumber, adapter.funcNumber)
 	cmd = raiddrivers.GetCommand("ls", pciDir, "|", "grep", "host")
@@ -601,18 +607,24 @@ func (adapter *MegaRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage
 
 type StorcliAdaptor struct {
 	Controller int
-	SN         string
+	sn         string
+	name       string
 }
 
 func newStorcliAdaptor() *StorcliAdaptor {
 	return &StorcliAdaptor{
 		Controller: -1,
-		SN:         "",
+		sn:         "",
+		name:       "",
 	}
 }
 
+func (a StorcliAdaptor) key() string {
+	return a.name + a.sn
+}
+
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.SN != ""
+	return a.Controller >= 0 && a.key() != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {
@@ -626,13 +638,15 @@ func (a *StorcliAdaptor) parseLine(l string) {
 	case "Controller":
 		a.Controller, _ = strconv.Atoi(val)
 	case "Serial Number":
-		a.SN = val
+		a.sn = val
+	case "Product Name":
+		a.name = val
 	}
 }
 
 func (raid *MegaRaid) GetStorcliAdaptor() (map[string]*StorcliAdaptor, error) {
 	ret := make(map[string]*StorcliAdaptor)
-	cmd := GetCommand2("/call", "show", "|", "grep", "-iE", `'^(Controller|Serial Number)\s='`)
+	cmd := GetCommand2("/call", "show", "|", "grep", "-iE", `'^(Controller|Product Name|Serial Number)\s='`)
 	lines, err := raid.term.Run(cmd)
 	if err != nil {
 		return nil, errors.Wrap(err, "Get storcli adapter")
@@ -641,7 +655,7 @@ func (raid *MegaRaid) GetStorcliAdaptor() (map[string]*StorcliAdaptor, error) {
 	for _, l := range lines {
 		adapter.parseLine(l)
 		if adapter.isComplete() {
-			ret[adapter.SN] = adapter
+			ret[adapter.key()] = adapter
 			adapter = newStorcliAdaptor()
 		}
 	}
@@ -656,9 +670,9 @@ func (adapter *MegaRaidAdaptor) storcliCtrlIndex() (int, error) {
 	if err != nil {
 		return -1, errors.Wrap(err, "Get all Storcli adaptor")
 	}
-	storAdap, ok := storcliAdaps[adapter.sn]
+	storAdap, ok := storcliAdaps[adapter.key()]
 	if !ok {
-		return -1, errors.Errorf("Not found storcli adaptor by SN %q", adapter.sn)
+		return -1, errors.Errorf("Not found storcli adaptor by SN %q", adapter.key())
 	}
 	return storAdap.Controller, nil
 }
@@ -914,6 +928,9 @@ func (raid *MegaRaid) ParsePhyDevs() error {
 	if err != nil {
 		return fmt.Errorf("List raid disk error: %v", err)
 	}
+	if raiddrivers.Debug {
+		log.Debugf("-PDList -aALL: %s", ret)
+	}
 	err = raid.parsePhyDevs(ret)
 	if err != nil {
 		return fmt.Errorf("parse physical disk device error: %v", err)

pkg/baremetal/utils/raid/raid.go

@@ -28,6 +28,10 @@ import (
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
+var (
+	Debug bool
+)
+
 const (
 	MODULE_MEGARAID = "megaraid_sas"
 	MODULE_HPSA     = "hpsa"

pkg/cloudcommon/agent/agent.go

@@ -239,6 +239,7 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
 	if agent.IAgent().GetAgentType() != string(api.AgentTypeEsxi) {
 		params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
 	}
+	params.Add(jsonutils.NewString(agent.IAgent().GetZoneName()), "zone_id")
 	params.Add(jsonutils.NewString(agent.IAgent().GetAgentType()), "agent_type")
 	ret, err := modules.Baremetalagents.List(session, params)
 	if err != nil {

pkg/cloudcommon/db/domain.go

@@ -106,9 +106,16 @@ func (manager *SDomainizedResourceBaseManager) ListItemFilter(
 	query apis.DomainizedResourceListInput,
 ) (*sqlchemy.SQuery, error) {
 	if len(query.ProjectDomains) > 0 {
+		// make sure ids are not utf8 string
+		idList := make([]string, 0)
+		for _, pid := range query.ProjectDomains {
+			if !stringutils2.IsUtf8(pid) {
+				idList = append(idList, pid)
+			}
+		}
 		tenants := TenantCacheManager.GetDomainQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(tenants.Field("id"), query.ProjectDomains),
+			sqlchemy.In(tenants.Field("id"), idList),
 			sqlchemy.In(tenants.Field("name"), query.ProjectDomains),
 		)).SubQuery()
 		q = q.In("domain_id", subq)

pkg/cloudcommon/db/project.go

@@ -101,9 +101,16 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SDomainizedResourceBaseManager.ListItemFilter")
 	}
 	if len(query.Projects) > 0 {
+		// make sure ids are not utf8 string
+		idList := make([]string, 0)
+		for _, pid := range query.Projects {
+			if !stringutils2.IsUtf8(pid) {
+				idList = append(idList, pid)
+			}
+		}
 		tenants := TenantCacheManager.GetTenantQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(tenants.Field("id"), query.Projects),
+			sqlchemy.In(tenants.Field("id"), idList),
 			sqlchemy.In(tenants.Field("name"), query.Projects),
 		)).SubQuery()
 		q = q.In("tenant_id", subq)

pkg/cloudcommon/db/taskman/subtasks.go

@@ -46,7 +46,7 @@ type SSubTask struct {
 	Stage     string `width:"64" charset:"ascii" nullable:"false" primary:"true"` // Column(VARCHAR(64, charset='ascii'), nullable=False, primary_key=True)
 	SubtaskId string `width:"36" charset:"ascii" nullable:"false" primary:"true"` // Column(VARCHAR(36, charset='ascii'), nullable=False, primary_key=True)
 	Status    string `width:"36" charset:"ascii" nullable:"false" default:"init"` // Column(VARCHAR(36, charset='ascii'), nullable=False, default=SUBTASK_INIT)
-	Result    string `length:"medium" charset:"ascii" nullable:"true"`            // Column(MEDIUMTEXT(charset='ascii'), nullable=True)
+	Result    string `length:"medium" charset:"utf8" nullable:"true"`             // Column(MEDIUMTEXT(charset='ascii'), nullable=True)
 }
 
 func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SSubTask {

pkg/cloudprovider/resources.go

@@ -487,6 +487,7 @@ type ICloudHostNetInterface interface {
 	GetIpAddr() string
 	GetMtu() int32
 	GetNicType() string
+	GetBridge() string
 }
 
 type ICloudLoadbalancer interface {

pkg/compute/models/dbinstances.go

@@ -1299,6 +1299,13 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("dbInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SDBInstance, 0)
 	commondb := make([]SDBInstance, 0)
 	commonext := make([]cloudprovider.ICloudDBInstance, 0)

pkg/compute/models/disks.go

@@ -1079,15 +1079,15 @@ func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) er
 		}
 		host := storage.GetMasterHost()
 		if host == nil {
-			return httperrors.NewBadRequestError("storage of disk no valid host")
+			return httperrors.NewBadRequestError("storage of disk %s no valid host", self.Id)
 		}
 	}
 	cnt, err := self.GetGuestDiskCount()
 	if err != nil {
-		return httperrors.NewInternalServerError("GetGuestDiskCount fail %s", err)
+		return httperrors.NewInternalServerError("GetGuestDiskCount for disk %s fail %s", self.Id, err)
 	}
 	if cnt > 0 {
-		return httperrors.NewNotEmptyError("Virtual disk used by virtual servers")
+		return httperrors.NewNotEmptyError("Virtual disk %s(%s) used by virtual servers", self.Name, self.Id)
 	}
 	if !isPurge && self.IsValidPrePaid() {
 		return httperrors.NewForbiddenError("not allow to delete prepaid disk in valid status")

pkg/compute/models/elasticcache_instances.go

@@ -452,6 +452,13 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("ElasticCacheInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SElasticcache, 0)
 	commondb := make([]SElasticcache, 0)
 	commonext := make([]cloudprovider.ICloudElasticcache, 0)

pkg/compute/models/guest_actions.go

@@ -89,7 +89,7 @@ func (self *SGuest) PreCheckPerformAction(
 ) error {
 	if self.Hypervisor == api.HYPERVISOR_KVM {
 		host := self.GetHost()
-		if (host.HostStatus == api.HOST_OFFLINE || !host.Enabled.Bool()) &&
+		if host != nil && (host.HostStatus == api.HOST_OFFLINE || !host.Enabled.Bool()) &&
 			utils.IsInStringArray(action,
 				[]string{
 					"start", "restart", "stop", "reset", "rebuild-root",

pkg/compute/models/guestnetworks.go

@@ -644,6 +644,8 @@ func totalGuestNicCount(
 	q = q.Join(guests, sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id")))
 	q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id")))
 
+	q = q.Filter(sqlchemy.IsFalse(guests.Field("pending_deleted")))
+
 	q = CloudProviderFilter(q, hosts.Field("manager_id"), providers, brands, cloudEnv)
 	q = RangeObjectsFilter(q, rangeObjs, nil, hosts.Field("zone_id"), hosts.Field("manager_id"), hosts.Field("id"), nil)
 

pkg/compute/models/guests.go

@@ -2773,7 +2773,7 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host
 	if vnet == nil {
 		if vnic.InClassicNetwork() {
 			region := host.GetRegion()
-			cloudprovider := region.GetCloudprovider()
+			cloudprovider := host.GetCloudprovider()
 			vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, cloudprovider, region)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
@@ -3631,7 +3631,7 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To
 		disk := guestdisk.GetDisk()
 		err := guestdisk.Detach(ctx, userCred)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "guestdisk.Detach guest_id: %s disk_id: %s", guestdisk.GuestId, guestdisk.DiskId)
 		}
 
 		if disk != nil {

pkg/compute/models/hosts.go

@@ -674,7 +674,7 @@ func (self *SHost) RealDelete(ctx context.Context, userCred mcclient.TokenCreden
 		if storage != nil && storage.IsLocal() {
 			cnt, err := storage.GetDiskCount()
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "GetDiskCount")
 			}
 			if cnt > 0 {
 				return httperrors.NewNotEmptyError("Inconsistent: local storage is not empty???")
@@ -1418,6 +1418,17 @@ func (self *SHost) GetGuests() []SGuest {
 	return guests
 }
 
+func (self *SHost) GetKvmGuests() []SGuest {
+	q := GuestManager.Query().Equals("host_id", self.Id).Equals("hypervisor", api.HYPERVISOR_KVM)
+	guests := make([]SGuest, 0)
+	err := db.FetchModelObjects(GuestManager, q, &guests)
+	if err != nil {
+		log.Errorf("GetGuests %s", err)
+		return nil
+	}
+	return guests
+}
+
 func (self *SHost) GetGuestsMasterOnThisHost() []SGuest {
 	q := self.GetGuestsQuery().IsNotEmpty("backup_host_id")
 	guests := make([]SGuest, 0)
@@ -3519,6 +3530,9 @@ func (self *SHost) PerformOffline(ctx context.Context, userCred mcclient.TokenCr
 			if jsonutils.QueryBoolean(data, "update_health_status", false) {
 				self.EnableHealthCheck = false
 			}
+			// Note: update host status to unknown on host offline
+			// we did not have host status after host offline
+			self.Status = api.BAREMETAL_UNKNOWN
 			return nil
 		})
 		if err != nil {
@@ -4612,7 +4626,15 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 						enables = append(enables, extNics[i])
 					}
 				} else {
-					// do nothing, in sync
+					// in sync, sync interface and bridge
+					hw := host.getHostwireOfIdAndMac(netIfs[i].WireId, netIfs[i].Mac)
+					if hw != nil && (hw.Bridge != extNics[i].GetBridge() || hw.Interface != extNics[i].GetDevice()) {
+						db.Update(hw, func() error {
+							hw.Interface = extNics[i].GetDevice()
+							hw.Bridge = extNics[i].GetBridge()
+							return nil
+						})
+					}
 				}
 			} else {
 				reserveIp := false
@@ -4663,7 +4685,7 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 		// always try reserved pool
 		extNic := adds[i].netif
 		err = host.addNetif(ctx, userCred, extNic.GetMac(), "", extNic.GetIpAddr(), 0, extNic.GetNicType(), extNic.GetIndex(),
-			extNic.IsLinkUp(), int16(extNic.GetMtu()), false, "", "", true, true)
+			extNic.IsLinkUp(), int16(extNic.GetMtu()), false, extNic.GetDevice(), extNic.GetBridge(), true, true)
 		if err != nil {
 			result.AddError(err)
 		} else {
@@ -4896,7 +4918,7 @@ func (host *SHost) PerformHostMaintenance(ctx context.Context, userCred mcclient
 		preferHostId = host.Id
 	}
 
-	guests := host.GetGuests()
+	guests := host.GetKvmGuests()
 	for i := 0; i < len(guests); i++ {
 		lockman.LockObject(ctx, &guests[i])
 		defer lockman.ReleaseObject(ctx, &guests[i])

pkg/compute/models/loadbalancernetworks.go

@@ -285,6 +285,8 @@ func totalLBNicCount(
 	lbnics := LoadbalancernetworkManager.Query().SubQuery()
 	q := lbnics.Query()
 	q = q.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbnics.Field("loadbalancer_id")))
+	q = q.Filter(sqlchemy.IsFalse(lbs.Field("pending_deleted")))
+
 	switch scope {
 	case rbacutils.ScopeSystem:
 		// do nothing

pkg/compute/models/loadbalancers.go

@@ -732,6 +732,13 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred
 		return nil, nil, syncResult
 	}
 
+	for i := range dbLbs {
+		if taskman.TaskManager.IsInTask(&dbLbs[i]) {
+			syncResult.Error(fmt.Errorf("loadbalancer %s(%s)in task", dbLbs[i].Name, dbLbs[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := []SLoadbalancer{}
 	commondb := []SLoadbalancer{}
 	commonext := []cloudprovider.ICloudLoadbalancer{}

pkg/compute/models/networks.go

@@ -1,4 +1,3 @@
-// Copyright 2019 Yunion
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -354,7 +353,7 @@ func (manager *SNetworkManager) GetOrCreateClassicNetwork(ctx context.Context, w
 func (self *SNetwork) GetUsedAddresses() map[string]bool {
 	used := make(map[string]bool)
 
-	q := self.getUsedAddressQuery(true)
+	q := self.getUsedAddressQuery(nil, rbacutils.ScopeSystem, true)
 	results, err := q.AllStringMap()
 	if err != nil {
 		log.Errorf("GetUsedAddresses fail %s", err)
@@ -780,7 +779,7 @@ func (self *SNetwork) IsAddressInRange(address netutils.IPV4Addr) bool {
 }
 
 func (self *SNetwork) isAddressUsed(address string) (bool, error) {
-	q := self.getUsedAddressQuery(true)
+	q := self.getUsedAddressQuery(nil, rbacutils.ScopeSystem, true)
 	q = q.Equals("ip_addr", address)
 	count, err := q.CountWithError()
 	if err != nil && errors.Cause(err) != sql.ErrNoRows {
@@ -2426,7 +2425,7 @@ func (network *SNetwork) AllowGetDetailsAddresses(ctx context.Context, userCred
 	return network.IsOwner(userCred) || db.IsAdminAllowGetSpec(userCred, network, "addresses")
 }
 
-func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
+func (network *SNetwork) getUsedAddressQuery(owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope, addrOnly bool) *sqlchemy.SQuery {
 	guestnetworks := GuestnetworkManager.Query().Equals("network_id", network.Id).SubQuery()
 	var guestNetQ *sqlchemy.SQuery
 	if addrOnly {
@@ -2434,17 +2433,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			guestnetworks.Field("ip_addr"),
 		)
 	} else {
-		guests := GuestManager.Query().SubQuery()
+		guests := GuestManager.FilterByOwner(GuestManager.Query(), owner, scope).SubQuery()
 		guestNetQ = guestnetworks.Query(
 			guestnetworks.Field("ip_addr"),
 			guestnetworks.Field("mac_addr"),
 			sqlchemy.NewStringField(GuestManager.KeywordPlural()).Label("owner_type"),
-			guestnetworks.Field("guest_id").Label("owner_id"),
+			guests.Field("id").Label("owner_id"),
 			guests.Field("name").Label("owner"),
 			sqlchemy.NewStringField("").Label("associate_id"),
 			sqlchemy.NewStringField("").Label("associate_type"),
 			guestnetworks.Field("created_at"),
-		).Join(
+		).LeftJoin(
 			guests,
 			sqlchemy.Equals(
 				guests.Field("id"),
@@ -2460,17 +2459,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			groupnetworks.Field("ip_addr"),
 		)
 	} else {
-		groups := GroupManager.Query().SubQuery()
+		groups := GroupManager.FilterByOwner(GroupManager.Query(), owner, scope).SubQuery()
 		groupNetQ = groupnetworks.Query(
 			groupnetworks.Field("ip_addr"),
 			sqlchemy.NewStringField("").Label("mac_addr"),
 			sqlchemy.NewStringField(GroupManager.KeywordPlural()).Label("owner_type"),
-			groupnetworks.Field("group_id").Label("owner_id"),
+			groups.Field("id").Label("owner_id"),
 			groups.Field("name").Label("owner"),
 			sqlchemy.NewStringField("").Label("associate_id"),
 			sqlchemy.NewStringField("").Label("associate_type"),
 			groupnetworks.Field("created_at"),
-		).Join(
+		).LeftJoin(
 			groups,
 			sqlchemy.Equals(
 				groups.Field("id"),
@@ -2486,17 +2485,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			hostnetworks.Field("ip_addr"),
 		)
 	} else {
-		hosts := HostManager.Query().SubQuery()
+		hosts := HostManager.FilterByOwner(HostManager.Query(), owner, scope).SubQuery()
 		hostNetQ = hostnetworks.Query(
 			hostnetworks.Field("ip_addr"),
 			hostnetworks.Field("mac_addr"),
 			sqlchemy.NewStringField(HostManager.KeywordPlural()).Label("owner_type"),
-			hostnetworks.Field("baremetal_id").Label("owner_id"),
+			hosts.Field("id").Label("owner_id"),
 			hosts.Field("name").Label("owner"),
 			sqlchemy.NewStringField("").Label("associate_id"),
 			sqlchemy.NewStringField("").Label("associate_type"),
 			hostnetworks.Field("created_at"),
-		).Join(
+		).LeftJoin(
 			hosts,
 			sqlchemy.Equals(
 				hosts.Field("id"),
@@ -2535,17 +2534,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			lbnetworks.Field("ip_addr"),
 		)
 	} else {
-		loadbalancers := LoadbalancerManager.Query().SubQuery()
+		loadbalancers := LoadbalancerManager.FilterByOwner(LoadbalancerManager.Query(), owner, scope).SubQuery()
 		lbNetQ = lbnetworks.Query(
 			lbnetworks.Field("ip_addr"),
 			sqlchemy.NewStringField("").Label("mac_addr"),
 			sqlchemy.NewStringField(LoadbalancerManager.KeywordPlural()).Label("owner_type"),
-			lbnetworks.Field("loadbalancer_id").Label("owner_id"),
+			loadbalancers.Field("id").Label("owner_id"),
 			loadbalancers.Field("name").Label("owner"),
 			sqlchemy.NewStringField("").Label("associate_id"),
 			sqlchemy.NewStringField("").Label("associate_type"),
 			lbnetworks.Field("created_at"),
-		).Join(
+		).LeftJoin(
 			loadbalancers,
 			sqlchemy.Equals(
 				loadbalancers.Field("id"),
@@ -2555,6 +2554,7 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 	}
 
 	elasticips := ElasticipManager.Query().Equals("network_id", network.Id).SubQuery()
+	ownerEips := ElasticipManager.FilterByOwner(ElasticipManager.Query().Equals("network_id", network.Id), owner, scope).SubQuery()
 	var eipQ *sqlchemy.SQuery
 	if addrOnly {
 		eipQ = elasticips.Query(
@@ -2565,11 +2565,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			elasticips.Field("ip_addr"),
 			sqlchemy.NewStringField("").Label("mac_addr"),
 			sqlchemy.NewStringField(ElasticipManager.KeywordPlural()).Label("owner_type"),
-			elasticips.Field("id").Label("owner_id"),
-			elasticips.Field("name").Label("owner"),
-			elasticips.Field("associate_id"),
-			elasticips.Field("associate_type"),
+			ownerEips.Field("id").Label("owner_id"),
+			ownerEips.Field("name").Label("owner"),
+			ownerEips.Field("associate_id"),
+			ownerEips.Field("associate_type"),
 			elasticips.Field("created_at"),
+		).LeftJoin(
+			ownerEips,
+			sqlchemy.Equals(
+				elasticips.Field("id"),
+				ownerEips.Field("id"),
+			),
 		)
 	}
 
@@ -2580,17 +2586,17 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 			netifnetworks.Field("ip_addr"),
 		)
 	} else {
-		netifs := NetworkInterfaceManager.Query().SubQuery()
+		netifs := NetworkInterfaceManager.FilterByOwner(NetworkInterfaceManager.Query(), owner, scope).SubQuery()
 		netifsQ = netifnetworks.Query(
 			netifnetworks.Field("ip_addr"),
 			netifs.Field("mac").Label("mac_addr"),
 			sqlchemy.NewStringField(NetworkInterfaceManager.KeywordPlural()).Label("owner_type"),
-			netifnetworks.Field("networkinterface_id").Label("owner_id"),
+			netifs.Field("id").Label("owner_id"),
 			netifs.Field("name").Label("owner"),
 			netifs.Field("associate_id"),
 			netifs.Field("associate_type"),
 			netifnetworks.Field("created_at"),
-		).Join(
+		).LeftJoin(
 			netifs,
 			sqlchemy.Equals(
 				netifnetworks.Field("networkinterface_id"),
@@ -2599,7 +2605,33 @@ func (network *SNetwork) getUsedAddressQuery(addrOnly bool) *sqlchemy.SQuery {
 		)
 	}
 
-	return sqlchemy.Union(guestNetQ, groupNetQ, hostNetQ, reservedQ, lbNetQ, eipQ, netifsQ).Query()
+	dbnetworks := DBInstanceNetworkManager.Query().Equals("network_id", network.Id).SubQuery()
+	var dbNetQ *sqlchemy.SQuery
+	if addrOnly {
+		dbNetQ = dbnetworks.Query(
+			dbnetworks.Field("ip_addr"),
+		)
+	} else {
+		dbinstances := DBInstanceManager.FilterByOwner(DBInstanceManager.Query(), owner, scope).SubQuery()
+		dbNetQ = dbnetworks.Query(
+			dbnetworks.Field("ip_addr"),
+			sqlchemy.NewStringField("").Label("mac_addr"),
+			sqlchemy.NewStringField(DBInstanceManager.KeywordPlural()).Label("owner_type"),
+			dbinstances.Field("id").Label("owner_id"),
+			dbinstances.Field("name").Label("owner"),
+			sqlchemy.NewStringField("").Label("associate_id"),
+			sqlchemy.NewStringField("").Label("associate_type"),
+			dbnetworks.Field("created_at"),
+		).LeftJoin(
+			dbinstances,
+			sqlchemy.Equals(
+				dbinstances.Field("id"),
+				dbnetworks.Field("dbinstance_id"),
+			),
+		)
+	}
+
+	return sqlchemy.Union(guestNetQ, groupNetQ, hostNetQ, reservedQ, lbNetQ, eipQ, netifsQ, dbNetQ).Query()
 }
 
 type SNetworkAddressList []api.SNetworkAddress
@@ -2612,20 +2644,26 @@ func (a SNetworkAddressList) Less(i, j int) bool {
 	return ipI < ipJ
 }
 
-func (network *SNetwork) GetDetailsAddresses(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (network *SNetwork) GetDetailsAddresses(ctx context.Context, userCred mcclient.TokenCredential, input api.GetNetworkAddressesInput) (api.GetNetworkAddressesOutput, error) {
+	output := api.GetNetworkAddressesOutput{}
+
+	allowScope := policy.PolicyManager.AllowScope(userCred, api.SERVICE_TYPE, network.KeywordPlural(), policy.PolicyActionGet, "addresses")
+	scope := rbacutils.String2ScopeDefault(input.Scope, allowScope)
+	if scope.HigherThan(allowScope) {
+		return output, errors.Wrapf(httperrors.ErrNotSufficientPrivilege, "require %s allow %s", scope, allowScope)
+	}
 
 	netAddrs := make([]api.SNetworkAddress, 0)
-	q := network.getUsedAddressQuery(false)
+	q := network.getUsedAddressQuery(userCred, scope, false)
 	err := q.All(&netAddrs)
 	if err != nil {
-		return nil, httperrors.NewGeneralError(err)
+		return output, httperrors.NewGeneralError(err)
 	}
 
 	sort.Sort(SNetworkAddressList(netAddrs))
 
-	result := jsonutils.NewDict()
-	result.Add(jsonutils.Marshal(netAddrs), "addresses")
-	return result, nil
+	output.Addresses = netAddrs
+	return output, nil
 }
 
 func (net *SNetwork) AllowPerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {

pkg/compute/models/purge.go

@@ -82,14 +82,14 @@ func (host *SHost) purge(ctx context.Context, userCred mcclient.TokenCredential)
 
 	_, err := host.PerformDisable(ctx, userCred, nil, apis.PerformDisableInput{})
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "PerformDisable")
 	}
 
 	guests := host.GetGuests()
 	for i := range guests {
 		err := guests[i].purge(ctx, userCred)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "purge guest %s", guests[i].Id)
 		}
 	}
 
@@ -98,7 +98,7 @@ func (host *SHost) purge(ctx context.Context, userCred mcclient.TokenCredential)
 	for i := range storages {
 		err := storages[i].purgeDisks(ctx, userCred)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "purgeDisks for storage %s", storages[i].Name)
 		}
 	}
 
@@ -139,7 +139,7 @@ func (storage *SStorage) purgeDisks(ctx context.Context, userCred mcclient.Token
 	for i := range disks {
 		err := disks[i].purge(ctx, userCred)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "purge disk %s", disks[i].Id)
 		}
 	}
 	return nil

pkg/compute/models/scaling_trigger.go

@@ -171,10 +171,13 @@ func (st *SScalingTimer) ValidateCreateData(input api.ScalingPolicyCreateInput)
 	var err error
 	if input.TriggerType == api.TRIGGER_TIMING {
 		input.Timer, err = checkTimerCreateInput(input.Timer)
-		return input, err
+	} else {
+		input.CycleTimer, err = checkCycleTimerCreateInput(input.CycleTimer)
 	}
-	input.CycleTimer, err = checkCycleTimerCreateInput(input.CycleTimer)
-	return input, err
+	if err != nil {
+		return input, httperrors.NewInputParameterError(err.Error())
+	}
+	return input, nil
 }
 
 func (st *SScalingTimer) Register(ctx context.Context, userCred mcclient.TokenCredential) error {

pkg/compute/models/scheduled_tasks.go

@@ -178,7 +178,10 @@ func (stm *SScheduledTaskManager) ValidateCreateData(ctx context.Context, userCr
 	} else {
 		input.CycleTimer, err = checkCycleTimerCreateInput(input.CycleTimer)
 	}
-	return input, err
+	if err != nil {
+		return input, httperrors.NewInputParameterError(err.Error())
+	}
+	return input, nil
 }
 
 var wdsCN = []string{"", "一", "二", "三", "四", "五", "六", "日"}

pkg/compute/models/skus.go

@@ -97,7 +97,7 @@ type SServerSku struct {
 	OsName string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin" default:"Any"` // Windows|Linux|Any
 
 	SysDiskResizable tristate.TriState `default:"true" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
-	SysDiskType      string            `width:"32" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
+	SysDiskType      string            `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 	SysDiskMinSizeGB int               `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required。 windows比较新的版本都是50G左右。
 	SysDiskMaxSizeGB int               `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required
 

pkg/compute/models/skus_tools.go

@@ -72,9 +72,10 @@ func (self *SSkuResourcesMeta) GetDBInstanceSkusByRegionExternalId(regionExterna
 			return nil, errors.Wrapf(err, "obj.Unmarshal")
 		}
 		if len(sku.Zone1) > 0 {
-			zoneId := self.getZoneIdBySuffix(zoneMaps, sku.Zone1)
+			zoneId := self.getZoneIdBySuffix(zoneMaps, sku.Zone1) // Huawei rds sku zone1 maybe is cn-north-4f
 			if len(zoneId) == 0 {
-				return nil, fmt.Errorf("invalid sku %s %s zone1: %s", sku.Id, sku.CloudregionId, sku.Zone1)
+				log.Errorf("invalid sku %s(%s) %s zone1: %s", sku.Name, sku.Id, sku.CloudregionId, sku.Zone1)
+				continue
 			}
 			sku.Zone1 = zoneId
 		}
@@ -82,7 +83,8 @@ func (self *SSkuResourcesMeta) GetDBInstanceSkusByRegionExternalId(regionExterna
 		if len(sku.Zone2) > 0 {
 			zoneId := self.getZoneIdBySuffix(zoneMaps, sku.Zone2)
 			if len(zoneId) == 0 {
-				return nil, fmt.Errorf("invalid sku %s %s zone2: %s", sku.Id, sku.CloudregionId, sku.Zone2)
+				log.Errorf("invalid sku %s(%s) %s zone2: %s", sku.Name, sku.Id, sku.CloudregionId, sku.Zone2)
+				continue
 			}
 			sku.Zone2 = zoneId
 		}
@@ -90,7 +92,8 @@ func (self *SSkuResourcesMeta) GetDBInstanceSkusByRegionExternalId(regionExterna
 		if len(sku.Zone3) > 0 {
 			zoneId := self.getZoneIdBySuffix(zoneMaps, sku.Zone3)
 			if len(zoneId) == 0 {
-				return nil, fmt.Errorf("invalid sku %s %s zone3: %s", sku.Id, sku.CloudregionId, sku.Zone3)
+				log.Errorf("invalid sku %s(%s) %s zone3: %s", sku.Name, sku.Id, sku.CloudregionId, sku.Zone3)
+				continue
 			}
 			sku.Zone3 = zoneId
 		}

pkg/compute/models/vpcs.go

@@ -194,7 +194,7 @@ func (manager *SVpcManager) getVpcExternalIdForClassicNetwork(regionId, cloudpro
 func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(ctx context.Context, cloudprovider *SCloudprovider, region *SCloudregion) (*SVpc, error) {
 	externalId := manager.getVpcExternalIdForClassicNetwork(region.Id, cloudprovider.Id)
 	_vpc, err := db.FetchByExternalIdAndManagerId(manager, externalId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
-		return q.Equals("manager_id", region.ManagerId)
+		return q.Equals("manager_id", cloudprovider.Id)
 	})
 	if err == nil {
 		return _vpc.(*SVpc), nil
@@ -211,7 +211,7 @@ func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(ctx context.Context,
 	vpc.SetEnabled(false)
 	vpc.Status = api.VPC_STATUS_UNAVAILABLE
 	vpc.ExternalId = externalId
-	vpc.ManagerId = region.ManagerId
+	vpc.ManagerId = cloudprovider.Id
 	err = manager.TableSpec().Insert(ctx, vpc)
 	if err != nil {
 		return nil, errors.Wrap(err, "Insert vpc for classic network")

pkg/compute/models/wires.go

@@ -451,7 +451,6 @@ func (manager *SWireManager) totalCountQ(
 	providers []string, brands []string, cloudEnv string,
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
-	pendingDeleted bool,
 ) *sqlchemy.SQuery {
 	guestsQ := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId)
 	guests := guestsQ.SubQuery()
@@ -476,19 +475,24 @@ func (manager *SWireManager) totalCountQ(
 	}
 	lbs := lbsQ.SubQuery()
 
+	dbsQ := filterByScopeOwnerId(DBInstanceManager.Query(), scope, ownerId)
+	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		dbsQ = CloudProviderFilter(dbsQ, dbsQ.Field("manager_id"), providers, brands, cloudEnv)
+	}
+	if len(rangeObjs) > 0 {
+		dbsQ = RangeObjectsFilter(dbsQ, rangeObjs, dbsQ.Field("cloudregion_id"), dbsQ.Field("zone_id"), dbsQ.Field("manager_id"), nil, nil)
+	}
+	dbs := dbsQ.SubQuery()
+
 	gNics := GuestnetworkManager.Query().SubQuery()
 	gNicQ := gNics.Query(
 		gNics.Field("network_id"),
 		sqlchemy.COUNT("gnic_count"),
+		sqlchemy.SUM("pending_deleted_gnic_count", guests.Field("pending_deleted")),
 	)
 	gNicQ = gNicQ.Join(guests, sqlchemy.Equals(guests.Field("id"), gNics.Field("guest_id")))
 	gNicQ = gNicQ.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id")))
 	gNicQ = gNicQ.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-	if pendingDeleted {
-		gNicQ = gNicQ.Filter(sqlchemy.IsTrue(guests.Field("pending_deleted")))
-	} else {
-		gNicQ = gNicQ.Filter(sqlchemy.IsFalse(guests.Field("pending_deleted")))
-	}
 
 	hNics := HostnetworkManager.Query().SubQuery()
 	hNicQ := hNics.Query(
@@ -517,33 +521,75 @@ func (manager *SWireManager) totalCountQ(
 		sqlchemy.COUNT("lbnic_count"),
 	)
 	lbNicQ = lbNicQ.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbNics.Field("loadbalancer_id")))
-	if pendingDeleted {
-		lbNicQ = lbNicQ.Filter(sqlchemy.IsTrue(lbs.Field("pending_deleted")))
-	} else {
-		lbNicQ = lbNicQ.Filter(sqlchemy.IsFalse(lbs.Field("pending_deleted")))
+	lbNicQ = lbNicQ.Filter(sqlchemy.IsFalse(lbs.Field("pending_deleted")))
+
+	eipNics := ElasticipManager.Query().IsNotEmpty("network_id").SubQuery()
+	eipNicQ := eipNics.Query(
+		eipNics.Field("network_id"),
+		sqlchemy.COUNT("eipnic_count"),
+	)
+	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		eipNicQ = CloudProviderFilter(eipNicQ, eipNicQ.Field("manager_id"), providers, brands, cloudEnv)
 	}
+	if len(rangeObjs) > 0 {
+		eipNicQ = RangeObjectsFilter(eipNicQ, rangeObjs, eipNicQ.Field("cloudregion_id"), nil, eipNicQ.Field("manager_id"), nil, nil)
+	}
+
+	netifsQ := NetworkInterfaceManager.Query()
+	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		netifsQ = CloudProviderFilter(netifsQ, netifsQ.Field("manager_id"), providers, brands, cloudEnv)
+	}
+	if len(rangeObjs) > 0 {
+		netifsQ = RangeObjectsFilter(netifsQ, rangeObjs, netifsQ.Field("cloudregion_id"), nil, netifsQ.Field("manager_id"), nil, nil)
+	}
+	netifs := netifsQ.SubQuery()
+	netifNics := NetworkinterfacenetworkManager.Query().SubQuery()
+	netifNicQ := netifNics.Query(
+		netifNics.Field("network_id"),
+		sqlchemy.COUNT("netifnic_count"),
+	)
+	netifNicQ = netifNicQ.Join(netifs, sqlchemy.Equals(netifNics.Field("networkinterface_id"), netifs.Field("id")))
+
+	dbNics := DBInstanceNetworkManager.Query().SubQuery()
+	dbNicQ := dbNics.Query(
+		dbNics.Field("network_id"),
+		sqlchemy.COUNT("dbnic_count"),
+	)
+	dbNicQ = dbNicQ.Join(dbs, sqlchemy.Equals(dbs.Field("id"), dbNics.Field("dbinstance_id")))
+	dbNicQ = dbNicQ.Filter(sqlchemy.IsFalse(dbs.Field("pending_deleted")))
 
 	gNicSQ := gNicQ.GroupBy(gNics.Field("network_id")).SubQuery()
 	hNicSQ := hNicQ.GroupBy(hNics.Field("network_id")).SubQuery()
 	revSQ := revQ.GroupBy(revIps.Field("network_id")).SubQuery()
 	grpNicSQ := grpNicQ.GroupBy(groupNics.Field("network_id")).SubQuery()
 	lbNicSQ := lbNicQ.GroupBy(lbNics.Field("network_id")).SubQuery()
+	eipNicSQ := eipNicQ.GroupBy(eipNics.Field("network_id")).SubQuery()
+	netifNicSQ := netifNicQ.GroupBy(netifNics.Field("network_id")).SubQuery()
+	dbNicSQ := dbNicQ.GroupBy(dbNics.Field("network_id")).SubQuery()
 
 	networks := NetworkManager.Query().SubQuery()
 	netQ := networks.Query(
 		networks.Field("wire_id"),
 		sqlchemy.COUNT("id").Label("net_count"),
-		sqlchemy.SUM("gnic_count", gNicQ.Field("gnic_count")),
-		sqlchemy.SUM("hnic_count", hNicQ.Field("hnic_count")),
-		sqlchemy.SUM("rev_count", revQ.Field("rnic_count")),
+		sqlchemy.SUM("gnic_count", gNicSQ.Field("gnic_count")),
+		sqlchemy.SUM("pending_deleted_gnic_count", gNicSQ.Field("pending_deleted_gnic_count")),
+		sqlchemy.SUM("hnic_count", hNicSQ.Field("hnic_count")),
+		sqlchemy.SUM("rev_count", revSQ.Field("rnic_count")),
 		sqlchemy.SUM("grpnic_count", grpNicSQ.Field("grpnic_count")),
 		sqlchemy.SUM("lbnic_count", lbNicSQ.Field("lbnic_count")),
+		sqlchemy.SUM("eipnic_count", eipNicSQ.Field("eipnic_count")),
+		sqlchemy.SUM("netifnic_count", netifNicSQ.Field("netifnic_count")),
+		sqlchemy.SUM("dbnic_count", dbNicSQ.Field("dbnic_count")),
 	)
 	netQ = netQ.LeftJoin(gNicSQ, sqlchemy.Equals(gNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(hNicSQ, sqlchemy.Equals(hNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(revSQ, sqlchemy.Equals(revSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(grpNicSQ, sqlchemy.Equals(grpNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(lbNicSQ, sqlchemy.Equals(lbNicSQ.Field("network_id"), networks.Field("id")))
+	netQ = netQ.LeftJoin(eipNicSQ, sqlchemy.Equals(eipNicSQ.Field("network_id"), networks.Field("id")))
+	netQ = netQ.LeftJoin(netifNicSQ, sqlchemy.Equals(netifNicSQ.Field("network_id"), networks.Field("id")))
+	netQ = netQ.LeftJoin(dbNicSQ, sqlchemy.Equals(dbNicSQ.Field("network_id"), networks.Field("id")))
+
 	netQ = netQ.GroupBy(networks.Field("wire_id"))
 	netSQ := netQ.SubQuery()
 
@@ -553,10 +599,14 @@ func (manager *SWireManager) totalCountQ(
 		sqlchemy.SUM("emulated_wires_count", wires.Field("is_emulated")),
 		sqlchemy.SUM("net_count", netSQ.Field("net_count")),
 		sqlchemy.SUM("guest_nic_count", netSQ.Field("gnic_count")),
+		sqlchemy.SUM("pending_deleted_guest_nic_count", netSQ.Field("pending_deleted_gnic_count")),
 		sqlchemy.SUM("host_nic_count", netSQ.Field("hnic_count")),
 		sqlchemy.SUM("reserved_count", netSQ.Field("rev_count")),
 		sqlchemy.SUM("group_nic_count", netSQ.Field("grpnic_count")),
 		sqlchemy.SUM("lb_nic_count", netSQ.Field("lbnic_count")),
+		sqlchemy.SUM("eip_nic_count", netSQ.Field("eipnic_count")),
+		sqlchemy.SUM("netif_nic_count", netSQ.Field("netifnic_count")),
+		sqlchemy.SUM("db_nic_count", netSQ.Field("dbnic_count")),
 	)
 	q = q.LeftJoin(netSQ, sqlchemy.Equals(wires.Field("id"), netSQ.Field("wire_id")))
 
@@ -589,10 +639,15 @@ type WiresCountStat struct {
 	ReservedCount      int
 	GroupNicCount      int
 	LbNicCount         int
+	EipNicCount        int
+	NetifNicCount      int
+	DbNicCount         int
+
+	PendingDeletedGuestNicCount int
 }
 
 func (wstat WiresCountStat) NicCount() int {
-	return wstat.GuestNicCount + wstat.HostNicCount + wstat.ReservedCount + wstat.GroupNicCount + wstat.LbNicCount
+	return wstat.GuestNicCount + wstat.HostNicCount + wstat.ReservedCount + wstat.GroupNicCount + wstat.LbNicCount + wstat.NetifNicCount + wstat.EipNicCount + wstat.DbNicCount
 }
 
 func (manager *SWireManager) TotalCount(
@@ -601,7 +656,6 @@ func (manager *SWireManager) TotalCount(
 	providers []string, brands []string, cloudEnv string,
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
-	pendingDeleted bool,
 ) WiresCountStat {
 	stat := WiresCountStat{}
 	err := manager.totalCountQ(
@@ -609,7 +663,6 @@ func (manager *SWireManager) TotalCount(
 		hostTypes,
 		providers, brands, cloudEnv,
 		scope, ownerId,
-		pendingDeleted,
 	).First(&stat)
 	if err != nil {
 		log.Errorf("Wire total count: %v", err)

pkg/compute/policy/defaults.go

@@ -311,7 +311,7 @@ var (
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
-				{
+				/*{
 					Service:  api.SERVICE_TYPE,
 					Resource: "networks",
 					Action:   PolicyActionGet,
@@ -323,6 +323,7 @@ var (
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
+				*/
 			},
 		},
 	}

pkg/compute/regiondrivers/huawei.go

@@ -127,6 +127,10 @@ func (self *SHuaWeiRegionDriver) ValidateCreateLoadbalancerData(ctx context.Cont
 			return nil, fmt.Errorf("eip external id is empty")
 		}
 
+		if eip.ManagerId != vpc.ManagerId {
+			return nil, httperrors.NewInputParameterError("eip's manager (%s(%s)) does not match vpc's(%s(%s)) (%s)", eip.GetName(), eip.GetId(), vpc.GetName(), vpc.GetId(), vpc.ManagerId)
+		}
+
 		data.Set("eip_id", jsonutils.NewString(eip.ExternalId))
 	}
 

pkg/compute/tasks/baremetal_delete_task.go

@@ -58,7 +58,12 @@ func (self *BaremetalDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneM
 }
 
 func (self *BaremetalDeleteTask) OnDeleteBaremetalComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) {
-	baremetal.RealDelete(ctx, self.UserCred)
+	err := baremetal.RealDelete(ctx, self.UserCred)
+	if err != nil {
+		log.Errorln("RealDelete fail %s", err)
+		self.OnFailure(ctx, baremetal, jsonutils.Marshal(err))
+		return
+	}
 	self.SetStageComplete(ctx, nil)
 }
 

pkg/compute/tasks/disk_delete_task.go

@@ -101,10 +101,13 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
 	)
 
 	storage = disk.GetStorage()
-	if storage != nil {
-		host = storage.GetMasterHost()
+	if storage == nil { // dirty data
+		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
 	}
 
+	host = storage.GetMasterHost()
+
 	isPurge := false
 	if (host == nil || !host.GetEnabled()) && jsonutils.QueryBoolean(self.Params, "purge", false) {
 		isPurge = true
@@ -112,21 +115,25 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
 	disk.SetStatus(self.UserCred, api.DISK_DEALLOC, "")
 	if isPurge {
 		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
+	}
+	if isNeed, _ := disk.IsNeedWaitSnapshotsDeleted(); isNeed { // for kvm rbd disk
+		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
+	}
+	if len(disk.BackupStorageId) > 0 {
+		self.SetStage("OnMasterStorageDeleteDiskComplete", nil)
 	} else {
-		if isNeed, _ := disk.IsNeedWaitSnapshotsDeleted(); isNeed {
-			self.OnGuestDiskDeleteComplete(ctx, disk, nil)
-			return
-		}
-		if len(disk.BackupStorageId) > 0 {
-			self.SetStage("OnMasterStorageDeleteDiskComplete", nil)
-		} else {
-			self.SetStage("OnGuestDiskDeleteComplete", nil)
-		}
-		if host == nil {
-			self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("fail to find master host"))
-		} else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self); err != nil {
-			self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
-		}
+		self.SetStage("OnGuestDiskDeleteComplete", nil)
+	}
+	if host == nil {
+		self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("fail to find master host"))
+		return
+	}
+	err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self)
+	if err != nil {
+		self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
+		return
 	}
 }
 

pkg/compute/tasks/guest_delete_backup_task.go

@@ -127,6 +127,10 @@ func (self *GuestDeleteBackupTask) OnDeleteBackupComplete(ctx context.Context, g
 }
 
 func (self *GuestDeleteBackupTask) TaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	guest.RemoveMetadata(ctx, "switch_backup", self.UserCred)
+	guest.RemoveMetadata(ctx, "switch_backup_count", self.UserCred)
+	guest.RemoveMetadata(ctx, "create_backup", self.UserCred)
+	guest.RemoveMetadata(ctx, "create_backup_count", self.UserCred)
 	self.OnDeleteBackupComplete(ctx, guest, data)
 	self.SetStageComplete(ctx, nil)
 }

pkg/compute/tasks/guest_delete_task.go

@@ -174,8 +174,12 @@ func (self *GuestDeleteTask) doClearSecurityGroupComplete(ctx context.Context, g
 	models.IsolatedDeviceManager.ReleaseDevicesOfGuest(ctx, guest, self.UserCred)
 	guest.RevokeAllSecgroups(ctx, self.UserCred)
 	// sync revoked secgroups to remote cloud
-	self.SetStage("OnSyncConfigComplete", nil)
-	guest.StartSyncTaskWithoutSyncstatus(ctx, self.UserCred, false, self.GetTaskId())
+	if jsonutils.QueryBoolean(self.Params, "purge", false) {
+		self.OnSyncConfigComplete(ctx, guest, nil)
+	} else {
+		self.SetStage("OnSyncConfigComplete", nil)
+		guest.StartSyncTaskWithoutSyncstatus(ctx, self.UserCred, false, self.GetTaskId())
+	}
 }
 
 func (self *GuestDeleteTask) OnSyncConfigComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/usages/handler.go

@@ -371,7 +371,7 @@ func getDomainGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr
 
 		BucketUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
-		nicsUsage(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
+		nicsUsage("domain", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
 
 		SnapshotUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
@@ -413,7 +413,7 @@ func getProjectGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityP
 		DisksUsage(getKey(scope, "pending_delete_disks"), rangeObjs, hostTypes, nil, providers, brands, cloudEnv, scope, cred, true, false),
 		DisksUsage(getKey(scope, "pending_delete_disks.system"), rangeObjs, hostTypes, nil, providers, brands, cloudEnv, scope, cred, true, true),
 
-		nicsUsage(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
+		nicsUsage("", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
 
 		SnapshotUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
@@ -601,35 +601,37 @@ func DisksUsage(
 
 func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) Usage {
 	count := make(map[string]interface{})
-	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil, false)
+	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil)
 	count["wires"] = result.WiresCount - result.EmulatedWiresCount
 	count["networks"] = result.NetCount
+	// include nics for pending_deleted guests
 	count["all.nics.guest"] = result.GuestNicCount
+	// nics for pending_deleted guests
+	count["all.nics.guest.pending_delete"] = result.PendingDeletedGuestNicCount
 	count["all.nics.host"] = result.HostNicCount
 	count["all.nics.reserve"] = result.ReservedCount
 	count["all.nics.group"] = result.GroupNicCount
 	count["all.nics.lb"] = result.LbNicCount
+	count["all.nics.eip"] = result.EipNicCount
+	count["all.nics.netif"] = result.NetifNicCount
+	count["all.nics.db"] = result.DbNicCount
 	count["all.nics"] = result.NicCount()
 
-	result = models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, rbacutils.ScopeSystem, nil, true)
-	count["all.nics.guest.pending_delete"] = result.GuestNicCount
-	count["all.nics.lb.pending_delete"] = result.LbNicCount
-	count["all.nics.pending_delete"] = result.GuestNicCount + result.LbNicCount
-
 	return count
 }
 
-func nicsUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage {
+func nicsUsage(prefix string, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage {
 	count := make(map[string]interface{})
-	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId, false)
-	count["nics.guest"] = result.GuestNicCount
-	count["nics.group"] = result.GroupNicCount
-	count["nics.lb"] = result.LbNicCount
-	count["nics"] = result.GuestNicCount + result.GroupNicCount + result.LbNicCount
-	result = models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId, true)
-	count["nics.guest.pending_delete"] = result.GuestNicCount
-	count["nics.lb.pending_delete"] = result.LbNicCount
-	count["nics.pending_delete"] = result.GuestNicCount + result.LbNicCount
+	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId)
+	// including nics for pending_deleted guests
+	count[prefixKey(prefix, "nics.guest")] = result.GuestNicCount
+	// #nics for pending_deleted guests
+	count[prefixKey(prefix, "nics.guest.pending_delete")] = result.PendingDeletedGuestNicCount
+	count[prefixKey(prefix, "nics.group")] = result.GroupNicCount
+	count[prefixKey(prefix, "nics.lb")] = result.LbNicCount
+	count[prefixKey(prefix, "nics.db")] = result.DbNicCount
+	count[prefixKey(prefix, "nics.eip")] = result.EipNicCount
+	count[prefixKey(prefix, "nics")] = result.GuestNicCount + result.GroupNicCount + result.LbNicCount + result.DbNicCount + result.EipNicCount
 	return count
 }
 

pkg/hostman/diskutils/disk.go

@@ -21,8 +21,8 @@ import (
 )
 
 type IDisk interface {
-	Connect() bool
-	Disconnect() bool
+	Connect() error
+	Disconnect() error
 	MountRootfs() fsdriver.IRootFsDriver
 	UmountRootfs(driver fsdriver.IRootFsDriver)
 }

pkg/hostman/diskutils/diskutils.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs"
@@ -62,15 +63,10 @@ func (d *SKVMGuestDisk) IsLVMPartition() bool {
 	return len(d.lvms) > 0
 }
 
-func (d *SKVMGuestDisk) ConnectWithoutDetectLvm() bool {
-	return d.connect()
-}
-
-func (d *SKVMGuestDisk) connect() bool {
+func (d *SKVMGuestDisk) connect() error {
 	d.nbdDev = nbd.GetNBDManager().AcquireNbddev()
 	if len(d.nbdDev) == 0 {
-		log.Errorln("Cannot get nbd device")
-		return false
+		return errors.Errorf("Cannot get nbd device")
 	}
 
 	var cmd []string
@@ -80,14 +76,14 @@ func (d *SKVMGuestDisk) connect() bool {
 			err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", "/etc/ceph").Run()
 			if err != nil {
 				log.Errorf("Failed to mkdir /etc/ceph: %s", err)
-				return false
+				return errors.Wrap(err, "Failed to mkdir /etc/ceph: %s")
 			}
 			err = procutils.NewRemoteCommandAsFarAsPossible("test", "-f", "/etc/ceph/ceph.conf").Run()
 			if err != nil {
 				err = procutils.NewRemoteCommandAsFarAsPossible("touch", "/etc/ceph/ceph.conf").Run()
 				if err != nil {
 					log.Errorf("failed to create /etc/ceph/ceph.conf: %s", err)
-					return false
+					return errors.Wrap(err, "failed to create /etc/ceph/ceph.conf")
 				}
 			}
 		}
@@ -95,10 +91,10 @@ func (d *SKVMGuestDisk) connect() bool {
 	} else {
 		cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, d.imagePath}
 	}
-	_, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output()
+	output, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output()
 	if err != nil {
-		log.Errorln(err.Error())
-		return false
+		log.Errorf("qemu-nbd connect failed %s %s", output, err.Error())
+		return errors.Wrapf(err, "qemu-nbd connect failed %s", output)
 	}
 
 	var tried uint = 0
@@ -107,31 +103,37 @@ func (d *SKVMGuestDisk) connect() bool {
 		err = d.findPartitions()
 		if err != nil {
 			log.Errorln(err.Error())
-			return false
+			return err
 		}
 		tried += 1
 	}
-	return true
+	return nil
 }
 
-func (d *SKVMGuestDisk) Connect() bool {
+func (d *SKVMGuestDisk) Connect() error {
 	pathType := d.connectionPrecheck()
 
-	if d.connect() == false {
-		return false
+	if err := d.connect(); err != nil {
+		return errors.Wrap(err, "disk.connect")
 	}
 
 	if pathType == LVM_PATH {
-		d.setupLVMS()
+		if _, err := d.setupLVMS(); err != nil {
+			return err
+		}
 	} else if pathType == PATH_TYPE_UNKNOWN {
 		hasLVM, err := d.setupLVMS()
+		if err != nil {
+			return err
+		}
+
 		// no lvm partition found and has partitions
-		if !hasLVM && err == nil && len(d.partitions) > 0 {
+		if !hasLVM && len(d.partitions) > 0 {
 			d.cacheNonLVMImagePath()
 		}
 	}
 
-	return true
+	return nil
 }
 
 func (d *SKVMGuestDisk) getImageFormat() string {
@@ -156,7 +158,7 @@ func (d *SKVMGuestDisk) findPartitions() error {
 	devpath := filepath.Dir(d.nbdDev)
 	files, err := ioutil.ReadDir(devpath)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "read dir %s", devpath)
 	}
 	for i := 0; i < len(files); i++ {
 		if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") {
@@ -264,30 +266,26 @@ func (d *SKVMGuestDisk) PutdownLVMs() {
 	d.lvms = []*SKVMGuestLVMPartition{}
 }
 
-func (d *SKVMGuestDisk) DisconnectWithoutLvm() bool {
-	return d.disconnect()
-}
-
-func (d *SKVMGuestDisk) Disconnect() bool {
+func (d *SKVMGuestDisk) Disconnect() error {
 	if len(d.nbdDev) > 0 {
 		defer d.LvmDisconnectNotify()
 		d.PutdownLVMs()
 		return d.disconnect()
 	} else {
-		return false
+		return nil
 	}
 }
 
-func (d *SKVMGuestDisk) disconnect() bool {
-	_, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "-d", d.nbdDev).Output()
+func (d *SKVMGuestDisk) disconnect() error {
+	output, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "-d", d.nbdDev).Output()
 	if err != nil {
 		log.Errorln(err.Error())
-		return false
+		return errors.Wrapf(err, "qemu-nbd disconnect %s", output)
 	}
 	nbd.GetNBDManager().ReleaseNbddev(d.nbdDev)
 	d.nbdDev = ""
 	d.partitions = d.partitions[len(d.partitions):]
-	return true
+	return nil
 
 }
 
@@ -347,6 +345,9 @@ func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) {
 }
 
 func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
+	if fd == nil {
+		return
+	}
 	d.UmountKvmRootfs(fd)
 }
 

pkg/hostman/diskutils/vddk.go

@@ -60,6 +60,8 @@ type VDDKDisk struct {
 	PartDirs []string
 	Proc     *Command
 	Pid      int
+
+	kvmDisk *SKVMGuestDisk
 }
 
 func NewVDDKDisk(vddkInfo *apis.VDDKConInfo, diskPath string) *VDDKDisk {
@@ -132,43 +134,36 @@ func logpath(pid int) string {
 	return fmt.Sprintf("%s/vixDiskLib-%d.log", TMPDIR, pid)
 }
 
-func (vd *VDDKDisk) Connect() bool {
-	return true
+func (vd *VDDKDisk) Connect() error {
+	flatFile, err := vd.ConnectBlockDevice()
+	if err != nil {
+		return err
+	}
+	vd.kvmDisk = NewKVMGuestDisk(flatFile)
+	return vd.kvmDisk.Connect()
 }
 
-func (vd *VDDKDisk) Disconnect() bool {
-	return true
+func (vd *VDDKDisk) Disconnect() error {
+	if vd.kvmDisk != nil {
+		if err := vd.kvmDisk.Disconnect(); err != nil {
+			log.Errorf("kvm disk disconnect failed %s", err)
+		}
+	}
+	return vd.DisconnectBlockDevice()
 }
 
 func (vd *VDDKDisk) MountRootfs() fsdriver.IRootFsDriver {
-	var err error
-	if err = vd.Mount(); err == nil {
-		for _, mntPath := range vd.PartDirs {
-			part := newVDDKPartition(mntPath)
-			if fs := guestfs.DetectRootFs(part); fs != nil {
-				log.Infof("Use rootfs %s", fs)
-				return fs
-			}
-		}
+	if vd.kvmDisk == nil {
+		return nil
 	}
-	if err != nil {
-		log.Errorf("VDDKDisk Mount failed: %s", err)
-	}
-	// something is wrong
-	if vd.Proc != nil {
-		err := vd.Proc.Kill()
-		if err != nil {
-			log.Errorf("unable to kill proc: %s", err.Error())
-		}
-	}
-	return nil
+	return vd.kvmDisk.MountRootfs()
 }
 
 func (vd *VDDKDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
-	err := vd.Umount()
-	if err != nil {
-		log.Errorf("VDDKDisk Umount failed: %s", err)
+	if vd.kvmDisk == nil {
+		return
 	}
+	vd.kvmDisk.UmountRootfs(fd)
 }
 
 func (vd *VDDKDisk) ParsePartitions(buf string) error {
@@ -275,6 +270,7 @@ func (vd *VDDKDisk) ExecProg() error {
 	}
 	cmd := NewCommand(execpath(), "-info", "-host", vd.Host, "-port", strconv.Itoa(vd.Port), "-user", vd.User,
 		"-password", vd.Passwd, "-mode", "nbd", "-thumb", thumb, "-vm", fmt.Sprintf("moref=%s", vd.VmRef), vd.DiskPath)
+	log.Debugf("command to mount: %s", cmd)
 	env := os.Environ()
 	env = append(env, fmt.Sprintf("LD_LIBRARY_PATH=%s", libdir()))
 	cmd.Env = env
@@ -321,7 +317,7 @@ func (vd *VDDKDisk) getServerCertThumbSha1(addr string) (string, error) {
 
 func (vd *VDDKDisk) WaitMounted() error {
 	endStr := []byte("Do you want to procede to unmount the volume")
-	timeout := 30 * time.Second
+	timeout := 300 * time.Second
 	endClock := time.After(timeout)
 	isEnd := false
 
@@ -349,13 +345,16 @@ Loop:
 	}
 	if vd.Proc.Exited() {
 		retCode := vd.Proc.ProcessState.ExitCode()
-		// ignore the error
-		vd.Proc.Kill()
-		vd.Proc = nil
+		err := vd.Proc.Kill()
+		if err != nil {
+			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
+		}
 		return errors.Error(fmt.Sprintf("VDDKDisk prog exit error(%d): %s", retCode, backup))
 	} else if !isEnd {
-		// timeout
-		vd.Proc.Kill()
+		err := vd.Proc.Kill()
+		if err != nil {
+			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
+		}
 		return errors.Error(fmt.Sprintf("VDDKDisk read timeout, program blocked"))
 	}
 	return nil

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -49,6 +49,8 @@ const CENTOS_VGNAME = "centos"
 
 type DeployerServer struct{}
 
+var _ deployapi.DeployAgentServer = &DeployerServer{}
+
 func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployParams,
 ) (res *deployapi.DeployGuestFsResponse, err error) {
 	// There will be some occasional unknown panic, so temporarily capture panic here.
@@ -63,22 +65,22 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 			res, err = nil, errors.Error(msg)
 		}
 	}()
-	log.Infof("Deploy guest fs on %s", req.DiskPath)
+	log.Infof("********* Deploy guest fs on %s", req.DiskPath)
 	var disk = diskutils.GetIDisk(req)
 	if len(req.GuestDesc.Hypervisor) == 0 {
 		req.GuestDesc.Hypervisor = comapi.HYPERVISOR_KVM
 	}
 	defer disk.Disconnect()
-	if !disk.Connect() {
-		log.Infof("Failed to connect %s disk", req.GuestDesc.Hypervisor)
+	if err := disk.Connect(); err != nil {
+		log.Infof("Failed to connect %s disk: %s", req.GuestDesc.Hypervisor, err)
 		return new(deployapi.DeployGuestFsResponse), nil
 	}
 	root := disk.MountRootfs()
+	defer disk.UmountRootfs(root)
 	if root == nil {
 		log.Infof("Failed mounting rootfs for %s disk", req.GuestDesc.Hypervisor)
 		return new(deployapi.DeployGuestFsResponse), nil
 	}
-	defer disk.UmountRootfs(root)
 
 	ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo)
 	if err != nil {
@@ -92,11 +94,11 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 
 func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams,
 ) (*deployapi.Empty, error) {
-	log.Infof("Resize fs on %s", req.DiskPath)
+	log.Infof("********* Resize fs on %s", req.DiskPath)
 	disk := diskutils.NewKVMGuestDisk(req.DiskPath)
 	defer disk.Disconnect()
-	if !disk.Connect() {
-		return new(deployapi.Empty), errors.Error("resize fs disk connect failed")
+	if err := disk.Connect(); err != nil {
+		return new(deployapi.Empty), errors.Wrap(err, "disk connect failed")
 	}
 
 	root := disk.MountKvmRootfs()
@@ -116,10 +118,10 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara
 }
 
 func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) {
-	log.Infof("Format fs on %s", req.DiskPath)
+	log.Infof("********* Format fs on %s", req.DiskPath)
 	gd := diskutils.NewKVMGuestDisk(req.DiskPath)
 	defer gd.Disconnect()
-	if gd.Connect() {
+	if err := gd.Connect(); err == nil {
 		if err := gd.MakePartition(req.FsFormat); err == nil {
 			err = gd.FormatPartition(req.FsFormat, req.Uuid)
 			if err != nil {
@@ -128,19 +130,23 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara
 		} else {
 			return new(deployapi.Empty), err
 		}
+	} else {
+		log.Errorf("failed connect kvm disk %s: %s", req.DiskPath, err)
 	}
 	return new(deployapi.Empty), nil
 }
 
 func (*DeployerServer) SaveToGlance(ctx context.Context, req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) {
-	log.Infof("%s save to glance", req.DiskPath)
+	log.Infof("********* %s save to glance", req.DiskPath)
 	var (
 		kvmDisk = diskutils.NewKVMGuestDisk(req.DiskPath)
 		osInfo  string
 		relInfo *deployapi.ReleaseInfo
 	)
 	defer kvmDisk.Disconnect()
-	if kvmDisk.Connect() {
+	if err := kvmDisk.Connect(); err != nil {
+		log.Errorf("failed connect kvm disk %s: %s", req.DiskPath, err)
+	} else {
 		var err error
 		func() {
 			if root := kvmDisk.MountKvmRootfs(); root != nil {
@@ -182,11 +188,11 @@ func getImageInfo(kvmDisk *diskutils.SKVMGuestDisk, rootfs fsdriver.IRootFsDrive
 }
 
 func (*DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) {
-	log.Infof("%s probe image info", req.DiskPath)
+	log.Infof("********* %s probe image info", req.DiskPath)
 	kvmDisk := diskutils.NewKVMGuestDisk(req.DiskPath)
 	defer kvmDisk.Disconnect()
-	if !kvmDisk.Connect() {
-		log.Infof("Failed to connect kvm disk")
+	if err := kvmDisk.Connect(); err != nil {
+		log.Infof("Failed to connect kvm disk %s: %s", req.DiskPath, err)
 		return new(deployapi.ImageInfo), errors.Error("Disk connector failed to connect image")
 	}
 
@@ -206,7 +212,7 @@ var connectedEsxiDisks = map[string]*diskutils.VDDKDisk{}
 func (*DeployerServer) ConnectEsxiDisks(
 	ctx context.Context, req *deployapi.ConnectEsxiDisksParams,
 ) (*deployapi.EsxiDisksConnectionInfo, error) {
-	log.Infof("Connect esxi disks ...")
+	log.Infof("********* Connect esxi disks ...")
 	var (
 		err          error
 		flatFilePath string
@@ -241,7 +247,7 @@ func (*DeployerServer) ConnectEsxiDisks(
 func (*DeployerServer) DisconnectEsxiDisks(
 	ctx context.Context, req *deployapi.EsxiDisksConnectionInfo,
 ) (*deployapi.Empty, error) {
-	log.Infof("Disconnect esxi disks ...")
+	log.Infof("********* Disconnect esxi disks ...")
 	for i := 0; i < len(req.Disks); i++ {
 		if disk, ok := connectedEsxiDisks[req.Disks[i].DiskPath]; ok {
 			if e := disk.DisconnectBlockDevice(); e != nil {

pkg/hostman/hostinfo/hostinfo.go

@@ -757,6 +757,7 @@ func (h *SHostInfo) tryCreateNetworkOnWire() {
 	params.Set("mask", jsonutils.NewInt(int64(mask)))
 	params.Set("is_on_premise", jsonutils.JSONTrue)
 	params.Set("server_type", jsonutils.NewString(api.NETWORK_TYPE_BAREMETAL))
+	params.Set("is_on_premise", jsonutils.JSONTrue)
 	ret, err := modules.Networks.PerformClassAction(
 		hostutils.GetComputeSession(context.Background()),
 		"try-create-network", params)
@@ -1295,6 +1296,9 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) {
 
 func (h *SHostInfo) uploadStorageInfo() {
 	for _, s := range storageman.GetManager().Storages {
+		if err := s.SetStorageInfo(s.GetId(), s.GetStorageName(), s.GetStorageConf()); err != nil {
+			h.onFail(err)
+		}
 		res, err := s.SyncStorageInfo()
 		if err != nil {
 			h.onFail(err)

pkg/hostman/storageman/storage_base.go

@@ -262,6 +262,7 @@ func (s *SBaseStorage) bindMountTo(sPath string) error {
 			return errors.Errorf("bind mount temp path to local image path failed %s", out)
 		}
 	}
+	log.Infof("bind mount %s -> %s", tempPath, sPath)
 	return nil
 }
 

pkg/hostman/storageman/storagehandler/storagehandler.go

@@ -66,7 +66,7 @@ func storageVerifyMountPoint(ctx context.Context, w http.ResponseWriter, r *http
 		hostutils.Response(ctx, w, httperrors.NewMissingParameterError("mount_point"))
 		return
 	}
-	output, err := procutils.NewCommand("mountpoint", mountPoint).Output()
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", mountPoint).Output()
 	if err == nil {
 		appsrv.SendStruct(w, map[string]interface{}{"is_mount_point": true})
 	} else {

pkg/image/models/images.go

@@ -603,6 +603,16 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		if appParams != nil && appParams.Request.ContentLength > 0 {
 			return nil, httperrors.NewInvalidStatusError("cannot upload in status %s", self.Status)
 		}
+		if minDiskSize, err := data.Int("min_disk"); err == nil {
+			img, err := qemuimg.NewQemuImage(self.getLocalLocation())
+			if err != nil {
+				return nil, errors.Wrap(err, "open image")
+			}
+			virtualSizeMB := img.SizeBytes / 1024 / 1024
+			if virtualSizeMB > 0 && minDiskSize < virtualSizeMB {
+				return nil, httperrors.NewBadRequestError("min disk size must >= %v", virtualSizeMB)
+			}
+		}
 	} else {
 		appParams := appsrv.AppContextGetParams(ctx)
 		if appParams != nil {
@@ -654,7 +664,6 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		return nil, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData")
 	}
 	data.Update(jsonutils.Marshal(input))
-
 	return data, nil
 }
 

pkg/image/policy/defaults.go

@@ -44,7 +44,7 @@ var (
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
-				{
+				/*{
 					Service:  api.SERVICE_TYPE,
 					Resource: "images",
 					Action:   PolicyActionList,
@@ -68,6 +68,7 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
+				*/
 			},
 		},
 		{

pkg/keystone/models/identitybase.go

@@ -132,9 +132,16 @@ func (manager *SIdentityBaseResourceManager) ListItemFilter(
 	}
 	// override manager.SDomainizedResourceBaseManager.ListItemFilter()
 	if len(query.ProjectDomains) > 0 {
+		// make sure ids are not utf8 string
+		idList := make([]string, 0)
+		for _, pid := range query.ProjectDomains {
+			if !stringutils2.IsUtf8(pid) {
+				idList = append(idList, pid)
+			}
+		}
 		domains := DomainManager.Query().SubQuery()
 		subq := domains.Query(domains.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(domains.Field("id"), query.ProjectDomains),
+			sqlchemy.In(domains.Field("id"), idList),
 			sqlchemy.In(domains.Field("name"), query.ProjectDomains),
 		))
 		q = q.In("domain_id", subq.SubQuery())

pkg/multicloud/aws/clouduser.go

@@ -174,6 +174,7 @@ func (self *SAwsClient) CreateClouduser(path string, username string) (*SCloudus
 }
 
 func (self *SAwsClient) DeleteClouduser(name string) error {
+	self.DeleteLoginProfile(name)
 	params := map[string]string{
 		"UserName": name,
 	}

pkg/multicloud/esxi/devtools.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/vmware/govmomi/vim25/types"
 
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 )
 
@@ -144,15 +145,25 @@ func NewUSBController(key *int32) types.BaseVirtualDevice {
 	return &device
 }
 
-func NewVNICDev(host *SHost, mac, driver string, vlanId int32, key, ctlKey, index int32) (types.BaseVirtualDevice, error) {
+func NewVNICDev(host *SHost, mac, driver string, bridge string, vlanId int32, key, ctlKey, index int32) (types.BaseVirtualDevice, error) {
 	desc := types.Description{Label: fmt.Sprintf("Network adapter %d", index+1), Summary: "VM Network"}
 
-	inet, err := host.FindNetworkByVlanID(vlanId)
-	if err != nil {
-		return nil, errors.Wrap(err, "SHost.FindNetworkByVlanID")
+	var inet IVMNetwork
+	var err error
+	if (vlanId == 0 || vlanId == 1) && len(bridge) > 0 {
+		inet, err = host.findDVPGById(bridge)
+		if err != nil {
+			log.Errorf("fail to find dvportgroup by name %s: %s", bridge, err)
+		}
 	}
 	if inet == nil || reflect.ValueOf(inet).IsNil() {
-		return nil, errors.Error(fmt.Sprintf("VLAN %d not found", vlanId))
+		inet, err = host.FindNetworkByVlanID(vlanId)
+		if err != nil {
+			log.Errorf("fail to find network by vlanid %d: %s", vlanId, err)
+		}
+	}
+	if inet == nil || reflect.ValueOf(inet).IsNil() {
+		return nil, errors.Error(fmt.Sprintf("Brige %s VLAN %d not found", bridge, vlanId))
 	}
 
 	var (

pkg/multicloud/esxi/host.go

@@ -84,13 +84,15 @@ type SHost struct {
 	multicloud.SHostBase
 	SManagedObject
 
+	masterIp string
+
 	nicInfo      []SHostNicInfo
 	storageInfo  []SHostStorageInfo
 	datastores   []cloudprovider.ICloudStorage
 	storageCache *SDatastoreImageCache
 	vms          []cloudprovider.ICloudVM
 	parent       *mo.ComputeResource
-	networks     []SNetwork
+	networks     []IVMNetwork
 	tempalteVMs  []*SVirtualMachine
 }
 
@@ -335,6 +337,31 @@ func (self *SHost) getNicInfo(debug bool) []SHostNicInfo {
 	return self.nicInfo
 }
 
+func mask2len(mask string) int8 {
+	maskAddr, _ := netutils.NewIPV4Addr(mask)
+	return netutils.Mask2Len(maskAddr)
+}
+
+func (self *SHost) isVnicAdmin(nic types.HostVirtualNic) bool {
+	if len(self.masterIp) > 0 {
+		if self.masterIp == nic.Spec.Ip.IpAddress {
+			return true
+		} else {
+			return false
+		}
+	}
+	exist, err := self.manager.IsHostIpExists(nic.Spec.Ip.IpAddress)
+	if err != nil {
+		log.Errorf("IsHostIpExists %s fail %s", nic.Spec.Ip.IpAddress, err)
+		return false
+	}
+	if exist {
+		self.masterIp = nic.Spec.Ip.IpAddress
+		return true
+	}
+	return false
+}
+
 func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo {
 	moHost := self.getHostSystem()
 
@@ -365,56 +392,47 @@ func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo {
 	if len(moHost.Config.Network.ConsoleVnic) > 0 {
 		vnics = append(vnics, moHost.Config.Network.ConsoleVnic...)
 	}
-	findMaster := false
+
 	for _, nic := range vnics {
 		mac := netutils.FormatMacAddr(nic.Spec.Mac)
 		pnic := findHostNicByMac(nicInfoList, mac)
 		if pnic != nil {
-			findMaster = true
+			// findMaster = true
 			pnic.IpAddr = nic.Spec.Ip.IpAddress
-			if nic.Spec.Portgroup == "Management Network" || nic.Spec.Portgroup == "Service Console" {
+			pnic.IpAddrPrefixLen = mask2len(nic.Spec.Ip.SubnetMask)
+			if nic.Spec.Ip.IpV6Config != nil && len(nic.Spec.Ip.IpV6Config.IpV6Address) > 0 {
+				pnic.IpAddr6 = nic.Spec.Ip.IpV6Config.IpV6Address[0].IpAddress
+				pnic.IpAddr6PrefixLen = int8(nic.Spec.Ip.IpV6Config.IpV6Address[0].PrefixLength)
+			}
+			if self.isVnicAdmin(nic) {
 				pnic.NicType = api.NIC_TYPE_ADMIN
 			}
 			pnic.LinkUp = true
 			pnic.Mtu = nic.Spec.Mtu
-		}
-	}
-
-	if !findMaster && len(nicInfoList) > 0 {
-		// no match pnic found for master nic
-		// choose the first pnic
-		pnic := &nicInfoList[0]
-		for _, nic := range vnics {
-			if nic.Spec.Portgroup == "Management Network" || nic.Spec.Portgroup == "Service Console" {
-				pnic.NicType = api.NIC_TYPE_ADMIN
-				pnic.IpAddr = nic.Spec.Ip.IpAddress
-				pnic.LinkUp = true
-				pnic.Mtu = nic.Spec.Mtu
-				break
+			if nic.Spec.DistributedVirtualPort != nil {
+				pnic.DVPortGroup = nic.Spec.DistributedVirtualPort.PortgroupKey
 			}
-		}
-		if len(pnic.IpAddr) == 0 {
-			// find default route vnic
-			defRouteDev := make([]string, 0)
-			for _, r := range moHost.Config.Network.RouteTableInfo.IpRoute {
-				if r.Network == "0.0.0.0" && r.PrefixLength == 0 {
-					// default route
-					defRouteDev = append(defRouteDev, r.DeviceName)
-				}
+		} else {
+			info := SHostNicInfo{}
+			info.Dev = nic.Device
+			info.Driver = "vmkernel"
+			info.Mac = mac
+			info.Index = int8(len(nicInfoList))
+			info.LinkUp = true
+			info.IpAddr = nic.Spec.Ip.IpAddress
+			info.IpAddrPrefixLen = mask2len(nic.Spec.Ip.SubnetMask)
+			if nic.Spec.Ip.IpV6Config != nil && len(nic.Spec.Ip.IpV6Config.IpV6Address) > 0 {
+				info.IpAddr6 = nic.Spec.Ip.IpV6Config.IpV6Address[0].IpAddress
+				info.IpAddr6PrefixLen = int8(nic.Spec.Ip.IpV6Config.IpV6Address[0].PrefixLength)
 			}
-			if len(defRouteDev) == 1 {
-				for _, nic := range vnics {
-					if nic.Device == defRouteDev[0] {
-						pnic.NicType = api.NIC_TYPE_ADMIN
-						pnic.IpAddr = nic.Spec.Ip.IpAddress
-						pnic.LinkUp = true
-						pnic.Mtu = nic.Spec.Mtu
-						break
-					}
-				}
-			} else {
-				log.Errorf("find default route interfaces fail: %s", defRouteDev)
+			info.Mtu = nic.Spec.Mtu
+			if self.isVnicAdmin(nic) {
+				info.NicType = api.NIC_TYPE_ADMIN
 			}
+			if nic.Spec.DistributedVirtualPort != nil {
+				info.DVPortGroup = nic.Spec.DistributedVirtualPort.PortgroupKey
+			}
+			nicInfoList = append(nicInfoList, info)
 		}
 	}
 
@@ -819,6 +837,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat
 	for _, nic := range nics {
 		index, _ := nic.Int("index")
 		mac, _ := nic.GetString("mac")
+		bridge, _ := nic.GetString("bridge")
 		driver := "e1000"
 		if nic.Contains("driver") {
 			driver, _ = nic.GetString("driver")
@@ -830,7 +849,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat
 		if nic.Contains("vlan") {
 			vlanId, _ = nic.Int("vlan")
 		}
-		dev, err := NewVNICDev(self, mac, driver, int32(vlanId), 4000, 100, int32(index))
+		dev, err := NewVNICDev(self, mac, driver, bridge, int32(vlanId), 4000, 100, int32(index))
 		if err != nil {
 			return nil, errors.Wrap(err, "NewVNICDev")
 		}
@@ -888,6 +907,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 		for _, nic := range nics {
 			index, _ := nic.Int("index")
 			mac, _ := nic.GetString("mac")
+			bridge, _ := nic.GetString("bridge")
 			driver := "e1000"
 			if nic.Contains("driver") {
 				driver, _ = nic.GetString("driver")
@@ -899,7 +919,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 			if nic.Contains("vlan") {
 				vlanId, _ = nic.Int("vlan")
 			}
-			dev, err := NewVNICDev(host, mac, driver, int32(vlanId), 4000, 100, int32(index))
+			dev, err := NewVNICDev(host, mac, driver, bridge, int32(vlanId), 4000, 100, int32(index))
 			if err != nil {
 				return nil, errors.Wrap(err, "NewVNICDev")
 			}
@@ -1258,8 +1278,8 @@ func (host *SHost) IsActiveVlanID(vlanID int32) bool {
 	return false
 }
 
-func (host *SHost) findBasicNetwork(vlanID int32) (*SNetwork, error) {
-	nets, err := host.GetNetwork()
+func (host *SHost) findBasicNetwork(vlanID int32) (IVMNetwork, error) {
+	nets, err := host.getBasicNetworks()
 	if err != nil {
 		return nil, err
 	}
@@ -1267,35 +1287,76 @@ func (host *SHost) findBasicNetwork(vlanID int32) (*SNetwork, error) {
 		return nil, nil
 	}
 	if !host.IsActiveVlanID(vlanID) {
-		return &nets[0], nil
+		return nets[0], nil
 	}
 	for i := range nets {
 		if nets[i].GetVlanId() == vlanID {
-			return &nets[i], nil
+			return nets[i], nil
 		}
 	}
 	return nil, nil
 }
 
-func (host *SHost) GetNetwork() ([]SNetwork, error) {
+func (host *SHost) getBasicNetworks() ([]IVMNetwork, error) {
+	nets, err := host.GetNetworks()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetNetworks")
+	}
+	ret := make([]IVMNetwork, 0)
+	for i := range nets {
+		if net, ok := nets[i].(*SNetwork); ok {
+			ret = append(ret, net)
+		}
+	}
+	return ret, nil
+}
+
+func (host *SHost) GetNetworks() ([]IVMNetwork, error) {
 	if host.networks != nil {
 		return host.networks, nil
 	}
 	netMobs := host.getHostSystem().Network
-	moNets := make([]mo.Network, 0)
-	err := host.manager.references2Objects(netMobs, NETWORK_PROPS, &moNets)
-	if err != nil {
-		return nil, errors.Wrap(err, "references2Objects")
+	netPortMobs := make([]types.ManagedObjectReference, 0)
+	netNetMobs := make([]types.ManagedObjectReference, 0)
+
+	for i := range netMobs {
+		log.Debugf("type: %s value: %s", netMobs[i].Type, netMobs[i].Value)
+		if netMobs[i].Type == "DistributedVirtualPortgroup" {
+			netPortMobs = append(netPortMobs, netMobs[i])
+		} else {
+			netNetMobs = append(netNetMobs, netMobs[i])
+		}
 	}
-	nets := make([]SNetwork, len(moNets))
-	for i := range moNets {
-		nets[i] = *NewNetwork(host.manager, &moNets[i], host.datacenter)
+
+	nets := make([]IVMNetwork, 0)
+
+	if len(netPortMobs) > 0 {
+		moPorts := make([]mo.DistributedVirtualPortgroup, 0)
+		err := host.manager.references2Objects(netPortMobs, DVPORTGROUP_PROPS, &moPorts)
+		if err != nil {
+			return nil, errors.Wrap(err, "references2Objects")
+		}
+		for i := range moPorts {
+			port := NewDistributedVirtualPortgroup(host.manager, &moPorts[i], host.datacenter)
+			nets = append(nets, port)
+		}
+	}
+	if len(netNetMobs) > 0 {
+		moNets := make([]mo.Network, 0)
+		err := host.manager.references2Objects(netNetMobs, NETWORK_PROPS, &moNets)
+		if err != nil {
+			return nil, errors.Wrap(err, "references2Objects")
+		}
+		for i := range moNets {
+			net := NewNetwork(host.manager, &moNets[i], host.datacenter)
+			nets = append(nets, net)
+		}
 	}
 
 	// network map
-	netMap := make(map[string]*SNetwork)
+	netMap := make(map[string]IVMNetwork)
 	for i := range nets {
-		netMap[nets[i].GetName()] = &nets[i]
+		netMap[nets[i].GetName()] = nets[i]
 	}
 
 	// fetch all portgroup
@@ -1306,7 +1367,7 @@ func (host *SHost) GetNetwork() ([]SNetwork, error) {
 			log.Infof("SNetwork corresponding to the portgroup whose name is %s could not be found", pg.Spec.Name)
 			continue
 		}
-		net.HostPortGroup = pg
+		net.SetHostPortGroup(pg)
 	}
 	host.networks = nets
 	return host.networks, nil
@@ -1330,6 +1391,19 @@ func (host *SHost) findNovlanDVPG() (*SDistributedVirtualPortgroup, error) {
 	return nil, nil
 }
 
+func (host *SHost) findDVPGById(id string) (*SDistributedVirtualPortgroup, error) {
+	nets, err := host.datacenter.GetNetworks()
+	if err != nil {
+		return nil, errors.Wrap(err, "SHost.datacenter.GetNetworks")
+	}
+	for _, net := range nets {
+		if dvpg, ok := net.(*SDistributedVirtualPortgroup); ok && dvpg.GetId() == id {
+			return dvpg, nil
+		}
+	}
+	return nil, nil
+}
+
 func (host *SHost) findVlanDVPG(vlanId int32) (*SDistributedVirtualPortgroup, error) {
 	nets, err := host.datacenter.GetNetworks()
 	if err != nil {

pkg/multicloud/esxi/hostnic.go

@@ -25,8 +25,14 @@ type SHostNicInfo struct {
 	Index   int8
 	LinkUp  bool
 	IpAddr  string
+	IpAddr6 string
 	Mtu     int32
 	NicType string
+
+	DVPortGroup string
+
+	IpAddrPrefixLen  int8
+	IpAddr6PrefixLen int8
 }
 
 func (nic *SHostNicInfo) GetDevice() string {
@@ -56,6 +62,18 @@ func (nic *SHostNicInfo) GetIpAddr() string {
 	return nic.IpAddr
 }
 
+func (nic *SHostNicInfo) GetIpAddrPrefixLen() int8 {
+	return nic.IpAddrPrefixLen
+}
+
+func (nic *SHostNicInfo) GetIpAddr6() string {
+	return nic.IpAddr6
+}
+
+func (nic *SHostNicInfo) GetIpAddr6PrefixLen() int8 {
+	return nic.IpAddr6PrefixLen
+}
+
 func (nic *SHostNicInfo) GetMtu() int32 {
 	return nic.Mtu
 }
@@ -63,3 +81,11 @@ func (nic *SHostNicInfo) GetMtu() int32 {
 func (nic *SHostNicInfo) GetNicType() string {
 	return nic.NicType
 }
+
+func (nic *SHostNicInfo) GetDVPortGroup() string {
+	return nic.DVPortGroup
+}
+
+func (nic *SHostNicInfo) GetBridge() string {
+	return nic.DVPortGroup
+}

pkg/multicloud/esxi/manager.go

@@ -466,6 +466,20 @@ func (cli *SESXiClient) checkHostManagedByVCenter() error {
 	return nil
 }
 
+func (cli *SESXiClient) IsHostIpExists(hostIp string) (bool, error) {
+	searchIndex := object.NewSearchIndex(cli.client.Client)
+
+	hostRef, err := searchIndex.FindByIp(cli.context, nil, cli.getPrivateId(hostIp), false)
+	if err != nil {
+		log.Errorf("searchIndex.FindByIp fail %s", err)
+		return false, err
+	}
+	if hostRef == nil {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (cli *SESXiClient) FindHostByIp(hostIp string) (*SHost, error) {
 	searchIndex := object.NewSearchIndex(cli.client.Client)
 

pkg/multicloud/esxi/network.go

@@ -30,6 +30,7 @@ type IVMNetwork interface {
 	GetActivePorts() []string
 	GetType() string
 	ContainHost(host *SHost) bool
+	SetHostPortGroup(pg types.HostPortGroup)
 }
 
 const (
@@ -52,6 +53,7 @@ type SNetwork struct {
 
 type SDistributedVirtualPortgroup struct {
 	SManagedObject
+	HostPortGroup types.HostPortGroup
 }
 
 func NewNetwork(manager *SESXiClient, net *mo.Network, dc *SDatacenter) *SNetwork {
@@ -101,6 +103,10 @@ func (net *SNetwork) ContainHost(host *SHost) bool {
 	return false
 }
 
+func (net *SNetwork) SetHostPortGroup(pg types.HostPortGroup) {
+	net.HostPortGroup = pg
+}
+
 func (net *SDistributedVirtualPortgroup) getMODVPortgroup() *mo.DistributedVirtualPortgroup {
 	return net.object.(*mo.DistributedVirtualPortgroup)
 }
@@ -170,6 +176,10 @@ func (net *SDistributedVirtualPortgroup) ContainHost(host *SHost) bool {
 	return false
 }
 
+func (net *SDistributedVirtualPortgroup) SetHostPortGroup(pg types.HostPortGroup) {
+	net.HostPortGroup = pg
+}
+
 func (net *SDistributedVirtualPortgroup) Uplink() bool {
 	dvpg := net.getMODVPortgroup()
 	return *dvpg.Config.Uplink

pkg/multicloud/esxi/shell/host.go

@@ -82,7 +82,7 @@ func init() {
 		if err != nil {
 			return err
 		}
-		networks, err := host.GetNetwork()
+		networks, err := host.GetNetworks()
 		if err != nil {
 			return err
 		}

pkg/multicloud/esxi/storagecache.go

@@ -81,17 +81,25 @@ func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, err
 	ret := make([]cloudprovider.ICloudImage, 0, 2)
 
 	// get vm template with only one disk
-	tems, err := self.host.GetTemplateVMs()
+	ihosts, err := self.datastore.GetAttachedHosts()
 	if err != nil {
-		log.Errorf("fail to get templateVMs of host '%s' in SDatastoreImageCache.GetIImages", self.host.GetName())
-		return ret, nil
+		return nil, errors.Wrap(err, "SDatastore.GetAttachedHosts")
 	}
-	for _, tem := range tems {
-		// for now, add vm template with only one disk as cachedimage
-		if len(tem.vdisks) != 1 {
-			continue
+
+	for _, ihost := range ihosts {
+		host := ihost.(*SHost)
+		tems, err := host.GetTemplateVMs()
+		if err != nil {
+			log.Errorf("fail to get templateVMs of host '%s' in SDatastoreImageCache.GetIImages", self.host.GetName())
+			return ret, nil
+		}
+		for _, tem := range tems {
+			// for now, add vm template with only one disk as cachedimage
+			if len(tem.vdisks) != 1 {
+				continue
+			}
+			ret = append(ret, NewVMTemplate(tem, self))
 		}
-		ret = append(ret, NewVMTemplate(tem, self))
 	}
 
 	files, err := self.datastore.ListDir(ctx, IMAGE_CACHE_DIR_NAME)

pkg/multicloud/esxi/template.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -56,11 +57,22 @@ func (t *SVMTemplate) GetGlobalId() string {
 }
 
 func (t *SVMTemplate) GetStatus() string {
-	_, err := t.cache.host.GetTemplateVMById(t.uuid)
-	if errors.Cause(err) == cloudprovider.ErrNotFound {
+	ihosts, err := t.cache.datastore.GetAttachedHosts()
+	if err != nil {
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	}
-	return api.CACHED_IMAGE_STATUS_READY
+	for _, ihost := range ihosts {
+		host := ihost.(*SHost)
+		_, err := host.GetTemplateVMById(t.uuid)
+		if err == nil {
+			return api.CACHED_IMAGE_STATUS_READY
+		}
+		if errors.Cause(err) != cloudprovider.ErrNotFound {
+			log.Errorf("fail to find templatevm %q: %v", t.uuid, err)
+			return api.CACHED_IMAGE_STATUS_CACHE_FAILED
+		}
+	}
+	return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 }
 
 func (t *SVMTemplate) Refresh() error {

pkg/multicloud/esxi/vdisk.go

@@ -140,6 +140,113 @@ func (s *sVirtualDiskSparseVer2BackingInfo) GetDatastore() *types.ManagedObjectR
 	return s.info.Datastore
 }
 
+type sVirtualDiskRawDiskMappingVer1BackingInfo struct {
+	info *types.VirtualDiskRawDiskMappingVer1BackingInfo
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetParent() IDiskBackingInfo {
+	if s.info.Parent != nil {
+		return &sVirtualDiskRawDiskMappingVer1BackingInfo{
+			info: s.info.Parent,
+		}
+	}
+	return nil
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetUuid() string {
+	return s.info.Uuid
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetDiskMode() string {
+	return s.info.DiskMode
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetWriteThrough() bool {
+	return false
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetFileName() string {
+	return s.info.FileName
+}
+
+func (s *sVirtualDiskRawDiskMappingVer1BackingInfo) GetDatastore() *types.ManagedObjectReference {
+	return s.info.Datastore
+}
+
+type sVirtualDiskSparseVer1BackingInfo struct {
+	info *types.VirtualDiskSparseVer1BackingInfo
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetParent() IDiskBackingInfo {
+	if s.info.Parent != nil {
+		return &sVirtualDiskSparseVer1BackingInfo{
+			info: s.info.Parent,
+		}
+	}
+	return nil
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetUuid() string {
+	return s.info.Datastore.String() + s.info.FileName
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetDiskMode() string {
+	return s.info.DiskMode
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetWriteThrough() bool {
+	if s.info.WriteThrough != nil && *s.info.WriteThrough == true {
+		return true
+	} else {
+		return false
+	}
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetFileName() string {
+	return s.info.FileName
+}
+
+func (s *sVirtualDiskSparseVer1BackingInfo) GetDatastore() *types.ManagedObjectReference {
+	return s.info.Datastore
+}
+
+type sVirtualDiskFlatVer1BackingInfo struct {
+	info *types.VirtualDiskFlatVer1BackingInfo
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetParent() IDiskBackingInfo {
+	if s.info.Parent != nil {
+		return &sVirtualDiskFlatVer1BackingInfo{
+			info: s.info.Parent,
+		}
+	}
+	return nil
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetUuid() string {
+	return s.info.Datastore.String() + s.info.FileName
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetDiskMode() string {
+	return s.info.DiskMode
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetWriteThrough() bool {
+	if s.info.WriteThrough != nil && *s.info.WriteThrough == true {
+		return true
+	} else {
+		return false
+	}
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetFileName() string {
+	return s.info.FileName
+}
+
+func (s *sVirtualDiskFlatVer1BackingInfo) GetDatastore() *types.ManagedObjectReference {
+	return s.info.Datastore
+}
+
 func (disk *SVirtualDisk) getBackingInfo() IDiskBackingInfo {
 	backing := disk.getVirtualDisk().Backing
 	switch backing.(type) {
@@ -149,10 +256,19 @@ func (disk *SVirtualDisk) getBackingInfo() IDiskBackingInfo {
 		}
 	case *types.VirtualDeviceFileBackingInfo:
 	case *types.VirtualDiskFlatVer1BackingInfo:
+		return &sVirtualDiskFlatVer1BackingInfo{
+			info: backing.(*types.VirtualDiskFlatVer1BackingInfo),
+		}
 	case *types.VirtualDiskLocalPMemBackingInfo:
 	case *types.VirtualDiskRawDiskMappingVer1BackingInfo:
+		return &sVirtualDiskRawDiskMappingVer1BackingInfo{
+			info: backing.(*types.VirtualDiskRawDiskMappingVer1BackingInfo),
+		}
 	case *types.VirtualDiskSeSparseBackingInfo:
 	case *types.VirtualDiskSparseVer1BackingInfo:
+		return &sVirtualDiskSparseVer1BackingInfo{
+			info: backing.(*types.VirtualDiskSparseVer1BackingInfo),
+		}
 	case *types.VirtualDiskSparseVer2BackingInfo:
 		return &sVirtualDiskSparseVer2BackingInfo{
 			info: backing.(*types.VirtualDiskSparseVer2BackingInfo),

pkg/multicloud/openstack/flavor.go

@@ -222,7 +222,7 @@ func (flavor *SFlavor) GetSysDiskResizable() bool {
 }
 
 func (flavor *SFlavor) GetSysDiskType() string {
-	return "iscsi"
+	return ""
 }
 
 func (flavor *SFlavor) GetSysDiskMinSizeGB() int {
@@ -246,7 +246,7 @@ func (flavor *SFlavor) GetAttachedDiskCount() int {
 }
 
 func (flavor *SFlavor) GetDataDiskTypes() string {
-	return "iscsi"
+	return ""
 }
 
 func (flavor *SFlavor) GetDataDiskMaxCount() int {

pkg/multicloud/openstack/image.go

@@ -92,7 +92,11 @@ func (region *SRegion) GetImages(name string, status string, imageId string) ([]
 	}
 	images := []SImage{}
 	resource := "/v2/images"
+	marker := ""
 	for {
+		if len(marker) > 0 {
+			query.Set("marker", marker)
+		}
 		resp, err := region.imageList(resource, query)
 		if err != nil {
 			return nil, errors.Wrap(err, "imageList")
@@ -109,7 +113,17 @@ func (region *SRegion) GetImages(name string, status string, imageId string) ([]
 		if len(part.Next) == 0 {
 			break
 		}
-		resource = part.Next
+		if len(part.Next) > 0 {
+			href, err := url.Parse(part.Next)
+			if err != nil {
+				marker = ""
+			} else {
+				marker = href.Query().Get("marker")
+			}
+		}
+		if len(marker) == 0 {
+			break
+		}
 	}
 	return images, nil
 }

pkg/util/redfish/bmconsole/lenovo.go

@@ -0,0 +1,90 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bmconsole
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+)
+
+func (r *SBMCConsole) GetLenovoConsoleJNLP(ctx context.Context) (string, error) {
+	loginData := strings.Join([]string{
+		"user=" + url.QueryEscape(r.username),
+		"password=" + url.QueryEscape(r.password),
+	}, "&")
+
+	// cookie:
+	// _appwebSessionId_=09eb9a178d520d2c9fa1430dd355dc27; path=/; httponly; secure
+
+	cookies := make(map[string]string)
+	cookies["_appwebSessionId_"] = ""
+
+	// first do html login
+	postHdr := http.Header{}
+	postHdr.Set("Content-Type", "application/x-www-form-urlencoded")
+	postHdr.Set("Referer", fmt.Sprintf("https://%s/login.html", r.host))
+	setCookieHeader(postHdr, cookies)
+	hdr, _, err := r.RawRequest(ctx, httputils.POST, "/data/login", postHdr, []byte(loginData))
+	if err != nil {
+		return "", errors.Wrap(err, "r.FormPost Login")
+	}
+	for _, cookieHdr := range hdr["Set-Cookie"] {
+		parts := strings.Split(cookieHdr, ";")
+		if len(parts) > 0 {
+			pparts := strings.Split(parts[0], "=")
+			if len(pparts) > 1 {
+				cookies[pparts[0]] = pparts[1]
+			}
+		}
+	}
+
+	getHdr := http.Header{}
+	getHdr.Set("Referer", fmt.Sprintf("https://%s/bmctree.html", r.host))
+	setCookieHeader(getHdr, cookies)
+	_, launchResp, err := r.RawRequest(ctx, httputils.GET, "/vkvmLaunch.html", getHdr, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "Get vkvmLauch.html")
+	}
+
+	var token string
+	st3Pattern := regexp.MustCompile(`<input type="hidden" name="ST3" value="(\w+)">`)
+	matched := st3Pattern.FindAllStringSubmatch(string(launchResp), -1)
+	if len(matched) > 0 && len(matched[0]) > 1 {
+		token = matched[0][1]
+	}
+
+	if len(token) == 0 {
+		return "", errors.Wrap(httperrors.ErrBadRequest, "no valid ST3 token")
+	}
+
+	path := fmt.Sprintf("viewer.jnlp(%s@0@%d)", r.host, time.Now().UnixNano()/1000000)
+	body := "ST3=" + url.QueryEscape(token)
+	getHdr.Set("Referer", fmt.Sprintf("https://%s/vkvmLaunch.html", r.host))
+	_, rspBody, err := r.RawRequest(ctx, httputils.POST, path, getHdr, []byte(body))
+	if err != nil {
+		return "", errors.Wrapf(err, "r.RawGet %s", path)
+	}
+	return string(rspBody), nil
+}

pkg/util/redfish/bmconsole/supermicro.go

@@ -16,14 +16,12 @@ package bmconsole
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/timeutils"
 
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
@@ -44,6 +42,7 @@ func (r *SBMCConsole) GetSupermicroConsoleJNLP(ctx context.Context) (string, err
 	// first do html login
 	postHdr := http.Header{}
 	postHdr.Set("Content-Type", "application/x-www-form-urlencoded")
+	postHdr.Set("Referer", fmt.Sprintf("http://%s/", r.host))
 	setCookieHeader(postHdr, cookies)
 	hdr, _, err := r.RawRequest(ctx, httputils.POST, "/cgi/login.cgi", postHdr, []byte(loginData))
 	if err != nil {
@@ -60,11 +59,12 @@ func (r *SBMCConsole) GetSupermicroConsoleJNLP(ctx context.Context) (string, err
 	}
 
 	getHdr := http.Header{}
+	getHdr.Set("Referer", fmt.Sprintf("https://%s/cgi/url_redirect.cgi?url_name=man_ikvm", r.host))
 	setCookieHeader(getHdr, cookies)
 
-	now := time.Now()
+	// now := time.Now()
 	// fwtype=255&time_stamp=Thu%20Apr%2023%202020%2002%3A25%3A13%20GMT%2B0800%20(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)&_=
-	loginData = strings.Join([]string{
+	/*loginData = strings.Join([]string{
 		"fwtype=255",
 		"time_stamp=" + url.QueryEscape(timeutils.RFC2882Time(now)),
 		"_=",
@@ -78,8 +78,9 @@ func (r *SBMCConsole) GetSupermicroConsoleJNLP(ctx context.Context) (string, err
 	if r.isDebug {
 		log.Debugf("upgrade_process.cgi %s", rspBody)
 	}
+	*/
 
-	_, rspBody, err = r.RawRequest(ctx, httputils.GET, "/cgi/url_redirect.cgi?url_name=ikvm&url_type=jwsk", getHdr, nil)
+	_, rspBody, err := r.RawRequest(ctx, httputils.GET, "/cgi/url_redirect.cgi?url_name=ikvm&url_type=jwsk", getHdr, nil)
 	if err != nil {
 		return "", errors.Wrap(err, "r.RawGet")
 	}

pkg/vpcagent/models/modelset.go

@@ -84,9 +84,11 @@ func (ms Vpcs) joinNetworks(subEntries Networks) bool {
 	for subId, subEntry := range subEntries {
 		wire := subEntry.Wire
 		if wire == nil {
-			// ensured by vpcs.joinWires
+			// let it go.  By the time the subnet has externalId or
+			// managerId set, we will not receive updates from them
+			// anymore
 			log.Warningf("network %s(%s) has no wire", subEntry.Name, subEntry.Id)
-			correct = false
+			delete(subEntries, subId)
 			continue
 		}
 		id := wire.VpcId

pkg/vpcagent/ovn/keeper.go

@@ -439,10 +439,26 @@ func (keeper *OVNNorthboundKeeper) ClaimGuestnetwork(ctx context.Context, guestn
 	gnp := &ovn_nb.LogicalSwitchPort{
 		Name:          lportName,
 		Addresses:     []string{fmt.Sprintf("%s %s", guestnetwork.MacAddr, guestnetwork.IpAddr)},
-		PortSecurity:  []string{fmt.Sprintf("%s %s/%d", guestnetwork.MacAddr, guestnetwork.IpAddr, guestnetwork.Network.GuestIpMask)},
 		Dhcpv4Options: &dhcpOpt,
 		Options:       map[string]string{},
 	}
+	if guest.SrcMacCheck.IsFalse() {
+		gnp.Addresses = append(gnp.Addresses, "unknown")
+		// empty, not nil, as match condition
+		gnp.PortSecurity = []string{}
+	} else if guest.SrcIpCheck.IsFalse() {
+		gnp.PortSecurity = []string{
+			fmt.Sprintf("%s", guestnetwork.MacAddr),
+		}
+	} else {
+		gnp.PortSecurity = []string{
+			fmt.Sprintf("%s %s/%d",
+				guestnetwork.MacAddr,
+				guestnetwork.IpAddr,
+				guestnetwork.Network.GuestIpMask,
+			),
+		}
+	}
 
 	var qosVif []*ovn_nb.QoS
 	if bwMbps := guestnetwork.BwLimit; bwMbps > 0 {