Merge pull request #2526 from ioito/automated-cherry-pick-of-#2522-upstream-release-2.9.0

Automated cherry pick of #2522: fix: sync status after guest sync conf

Gopkg.lock

@@ -524,6 +524,14 @@
   revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
   version = "v1.4.7"
 
+[[projects]]
+  digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda"
+  name = "github.com/ghodss/yaml"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
+  version = "v1.0.0"
+
 [[projects]]
   branch = "master"
   digest = "1:36fe9527deed01d2a317617e59304eb2c4ce9f8a24115bcc5c2e37b3aee5bae4"
@@ -533,9 +541,10 @@
   revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
 
 [[projects]]
-  digest = "1:ff09f03ca12c4d23c28c596eb0738745e36c1cac111f62f64819210f40607cd2"
+  digest = "1:d5083934eb25e45d17f72ffa86cae3814f4a9d6c073c4f16b64147169b245606"
   name = "github.com/gin-gonic/gin"
   packages = [
+    ".",
     "binding",
     "json",
     "render",
@@ -1071,12 +1080,12 @@
   version = "v2.0.7"
 
 [[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
+  branch = "master"
+  digest = "1:deff41c4160cc5b487a1d1a69a58925f2d3ac11befa0692b50e76940594e20f5"
   name = "github.com/pkg/errors"
   packages = ["."]
   pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
+  revision = "27936f6d90f9c8e1145f11ed52ffffbfdb9e0af7"
 
 [[projects]]
   branch = "master"
@@ -1497,14 +1506,6 @@
   revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8"
   version = "v1.17.0"
 
-[[projects]]
-  digest = "1:2655728699af00d823b0a68bd7169fd241a30c04252f456b34421908701b9c3f"
-  name = "gopkg.in/gin-gonic/gin.v1"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
-  version = "v1.2"
-
 [[projects]]
   digest = "1:cbc72c4c4886a918d6ab4b95e347ffe259846260f99ebdd8a198c2331cf2b2e9"
   name = "gopkg.in/go-playground/validator.v8"
@@ -1705,28 +1706,29 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:53c214ff8c2fd57db0962140b9a5e57b01b63066e7de7fe6897a633d15b78248"
+  digest = "1:f044fe02305a360710bcb0626b870599537514da95c9d4f4cd31df0b4c017e5f"
   name = "yunion.io/x/jsonutils"
   packages = ["."]
   pruneopts = "UT"
-  revision = "028b9007e455103eabacf3e6d71b5ce5060c3984"
+  revision = "a964e1e8a05199146284389fe6ef2d1eac2744b4"
 
 [[projects]]
   branch = "master"
-  digest = "1:b098c5b757b7622dfeed71f9cdcf0621a42051b88967286b0e1bcfb8a910fc86"
+  digest = "1:a872813eff1f3d353a1e4e64d745ac07387ed52cbc4cc2f6180272a7df7f21d1"
   name = "yunion.io/x/log"
   packages = [
     ".",
     "hooks",
   ]
   pruneopts = "UT"
-  revision = "71a36f55dcfcfa2c8b2477c339e2c7ebd78c731d"
+  revision = "9f6483a7103dc7b5012ae7aad2c74728e02cf650"
 
 [[projects]]
   branch = "master"
-  digest = "1:30ca06466aebad8751130956f96267bc520af0991990522109d4250086e3ae83"
+  digest = "1:c09298fb61e83dae28b08222e20f31bd109f93c50a29517a17edcc5a5666bd1a"
   name = "yunion.io/x/pkg"
   packages = [
+    "errors",
     "gotypes",
     "prettytable",
     "trace",
@@ -1751,30 +1753,29 @@
     "util/stringutils",
     "util/timeutils",
     "util/trace",
-    "util/ttlpool",
     "util/version",
     "util/wait",
     "util/workqueue",
     "utils",
   ]
   pruneopts = "UT"
-  revision = "fa6430d86cfb0b2b3cc6ea20bab3b4618186234b"
+  revision = "b564cfdcc2247b3f6f6bc4e818b44f08d82dc660"
 
 [[projects]]
   branch = "master"
-  digest = "1:29f0a99ba079309ef8c82def5b9c452d0871837780ee702d7ae7c1ebe47478b0"
+  digest = "1:f7386c58a1e94eddbca9b8a95167ba7b4bc7d8dfea40a01261f7443757eafb25"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
   pruneopts = "UT"
-  revision = "5e43c9cfbb8eb2870882b87d3ab429062d69be2b"
+  revision = "cfa59aeabbfe47c910920a2e1d61e1eaddd92ec5"
 
 [[projects]]
   branch = "master"
-  digest = "1:cfa79b54c4ec8d1fe6b1599846b3975c50212f58114d807a6db69bbee5a16b7a"
+  digest = "1:d91515edf1e995676f635f39ed5445e45fd3d176c71546744b11cba7e0126421"
   name = "yunion.io/x/structarg"
   packages = ["."]
   pruneopts = "UT"
-  revision = "5b725f46d67547e7fb603b2118eddc5358ac2a39"
+  revision = "115bed041de33bdf3a5be5281d4a6b06607c82d8"
 
 [solve-meta]
   analyzer-name = "dep"
@@ -1835,6 +1836,7 @@
     "github.com/coredns/coredns/request",
     "github.com/fatih/color",
     "github.com/fsnotify/fsnotify",
+    "github.com/gin-gonic/gin",
     "github.com/go-sql-driver/mysql",
     "github.com/golang-plus/errors",
     "github.com/golang-plus/uuid",
@@ -1855,6 +1857,7 @@
     "github.com/moul/http2curl",
     "github.com/nelsonken/cos-go-sdk-v5/cos",
     "github.com/pierrec/lz4",
+    "github.com/pkg/errors",
     "github.com/serialx/hashring",
     "github.com/shirou/gopsutil/cpu",
     "github.com/shirou/gopsutil/host",
@@ -1882,7 +1885,6 @@
     "golang.org/x/net/bpf",
     "golang.org/x/net/ipv4",
     "golang.org/x/sys/unix",
-    "gopkg.in/gin-gonic/gin.v1",
     "gopkg.in/yaml.v2",
     "k8s.io/api/core/v1",
     "k8s.io/apimachinery/pkg/api/errors",
@@ -1895,6 +1897,8 @@
     "k8s.io/kubernetes/pkg/scheduler/api/v1",
     "yunion.io/x/jsonutils",
     "yunion.io/x/log",
+    "yunion.io/x/log/hooks",
+    "yunion.io/x/pkg/errors",
     "yunion.io/x/pkg/gotypes",
     "yunion.io/x/pkg/prettytable",
     "yunion.io/x/pkg/trace",
@@ -1917,7 +1921,6 @@
     "yunion.io/x/pkg/util/stringutils",
     "yunion.io/x/pkg/util/timeutils",
     "yunion.io/x/pkg/util/trace",
-    "yunion.io/x/pkg/util/ttlpool",
     "yunion.io/x/pkg/util/version",
     "yunion.io/x/pkg/util/wait",
     "yunion.io/x/pkg/util/workqueue",

Gopkg.toml

@@ -188,6 +188,10 @@
   branch = "master"
   name = "yunion.io/x/sqlchemy"
 
+[[constraint]]
+  branch = "master"
+  name = "github.com/pkg/errors"
+
 [[constraint]]
   branch = "master"
   name = "yunion.io/x/structarg"

README.md

@@ -1,3 +1,4 @@
 # Yunion OneCloud
 
 [![CircleCI](https://circleci.com/gh/yunionio/onecloud.svg?style=svg)](https://circleci.com/gh/yunionio/onecloud) [![Build Status](https://travis-ci.org/yunionio/onecloud.svg?branch=master)](https://travis-ci.org/yunionio/onecloud)
+[![Go Report Card](https://goreportcard.com/badge/github.com/yunionio/onecloud)](https://goreportcard.com/report/github.com/yunionio/onecloud)

build/baremetal-agent/vars

@@ -0,0 +1 @@
+DESCRIPTION="Baremetal Agent Command Line Utility"

build/host/vars

@@ -0,0 +1 @@
+DESCRIPTION="Yunion Host Command Line Utility"

build/region/root/opt/yunion/share/notify_templates/compute/content/SERVER_REBUILD_ROOT

@@ -1 +1 @@
-您的{{ .hypervisor }}主机{{ .name }}的系统盘已经重置成功，{{ if .account }}初始帐号为{{ .account }}，{{ end }}{{ if .keypair }}访问密钥为{{ .keypair }}，{{ end }}{{ if .password }}初始密码为{{ .password }}，{{ end }}请使用{{ if .windows }}远程桌面连接器(RDC){{ else }}SSH{{ end }}或控制面板控制台访问云主机。
+您的{{ .hypervisor }}主机{{ .name }}重装系统成功，{{ if .account }}初始帐号为{{ .account }}，{{ end }}{{ if .keypair }}访问密钥为{{ .keypair }}，{{ end }}{{ if .password }}初始密码为{{ .password }}，{{ end }}请使用{{ if .windows }}远程桌面连接器(RDC){{ else }}SSH{{ end }}或控制面板控制台访问云主机。

build/region/root/opt/yunion/share/notify_templates/compute/title/SERVER_REBUILD_ROOT

@@ -1 +1 @@
-{{ .hypervisor }}主机{{ .name }}的系统盘重置成功
+{{ .hypervisor }}主机{{ .name }}重装系统成功

cmd/baremetal-agent/main.go

@@ -16,8 +16,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/baremetal/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.New().StartService()
 }

cmd/climc/shell/cloudaccounts.go

@@ -156,6 +156,17 @@ func init() {
 		return nil
 	})
 
+	R(&options.SUcloudCloudAccountCreateOptions{}, "cloud-account-create-ucloud", "Create a Ucloud cloud account", func(s *mcclient.ClientSession, args *options.SUcloudCloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Ucloud"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudaccountUpdateOptions struct {
 		ID        string `help:"ID or Name of cloud account"`
 		Name      string `help:"New name to update"`

cmd/climc/shell/hosts.go

@@ -37,7 +37,7 @@ func init() {
 		Occupied  bool   `help:"show occupid host" json:"-"`
 		Enabled   bool   `help:"Show enabled host only" json:"-"`
 		Disabled  bool   `help:"Show disabled host only" json:"-"`
-		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei"`
+		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud"`
 		AnyMac    string `help:"Mac matches one of the host's interface"`
 
 		IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"`
@@ -48,6 +48,8 @@ func init() {
 
 		Hypervisor string `help:"filter hosts by hypervisor"`
 
+		StorageNotAttached bool `help:"List hosts not attach specified storage"`
+
 		options.BaseListOptions
 	}
 	R(&HostListOptions{}, "host-list", "List hosts", func(s *mcclient.ClientSession, opts *HostListOptions) error {
@@ -465,6 +467,7 @@ func init() {
 		MAC        string `help:"Default MAC address of baremetal"`
 		Rack       string `help:"Rack number of baremetal"`
 		Slots      string `help:"Slots number of baremetal"`
+		IpmiUser   string `help:"IPMI user name"`
 		IpmiPasswd string `help:"IPMI user password"`
 		IpmiAddr   string `help:"IPMI IP address"`
 	}
@@ -479,6 +482,9 @@ func init() {
 		if len(args.Slots) > 0 {
 			params.Add(jsonutils.NewString(args.Slots), "slots")
 		}
+		if len(args.IpmiUser) > 0 {
+			params.Add(jsonutils.NewString(args.IpmiUser), "ipmi_username")
+		}
 		if len(args.IpmiPasswd) > 0 {
 			params.Add(jsonutils.NewString(args.IpmiPasswd), "ipmi_password")
 		}

cmd/climc/shell/loadbalancerbackendgroups.go

@@ -81,4 +81,5 @@ func init() {
 		printObject(lbbackendgroup)
 		return nil
 	})
+
 }

cmd/climc/shell/loadbalancernetworks.go

@@ -29,7 +29,7 @@ func init() {
 		Network      string `help:"ID or Name of network"`
 		Ip           string `help:"search the IP address"`
 	}
-	R(&LoadbalancerNetworkListOptions{}, "loadbalancer-network-list", "List loadbalancer network pairs", func(s *mcclient.ClientSession, args *LoadbalancerNetworkListOptions) error {
+	R(&LoadbalancerNetworkListOptions{}, "lbnetwork-list", "List loadbalancer network pairs", func(s *mcclient.ClientSession, args *LoadbalancerNetworkListOptions) error {
 		var params *jsonutils.JSONDict
 		{
 			var err error
@@ -57,5 +57,4 @@ func init() {
 		printList(result, modules.Loadbalancernetworks.GetColumns(s))
 		return nil
 	})
-
 }

cmd/climc/shell/loadbalancers.go

@@ -98,4 +98,5 @@ func init() {
 		printObject(lb)
 		return nil
 	})
+
 }

cmd/climc/shell/meteralert.go

@@ -28,32 +28,23 @@ func init() {
 	 * 创建一条报警规则
 	 */
 	type MeterAlertCreateOptions struct {
-		TYPE       string  `help:"Alert rule type" choices:"balance|resFee|monthFee"`
-		THRESHOLD  float64 `help:"Threshold value of the metric"`
-		COMPARATOR string  `help:"Comparison operator for join expressions" choices:">|<|>=|<=|=|!="`
-		RECIPIENTS string  `help:"Comma separated recipient ID"`
-		LEVEL      string  `help:"Alert level" choices:"normal|important|fatal"`
-		CHANNEL    string  `help:"Ways to send an alarm" choices:"email|mobile"`
+		Type       string  `help:"Alert rule type" required:"true" choices:"balance|resFee|monthFee"`
+		Threshold  float64 `help:"Threshold value of the metric" required:"true"`
+		Comparator string  `help:"Comparison operator for join expressions" required:"true" choices:">|<|>=|<=|=|!="`
+		Recipients string  `help:"Comma separated recipient ID" required:"true"`
+		Level      string  `help:"Alert level" required:"true" choices:"normal|important|fatal"`
+		Channel    string  `help:"Ways to send an alarm" required:"true" choices:"email|mobile"`
 		Provider   string  `help:"Name of the cloud platform"`
 		AccountId  string  `help:"ID of the cloud platform"`
+		ProjectId  string  `help:"ID of the project" required:"true"`
 	}
 	R(&MeterAlertCreateOptions{}, "meteralert-create", "Create a meter alert rule", func(s *mcclient.ClientSession, args *MeterAlertCreateOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.TYPE), "type")
-		params.Add(jsonutils.NewFloat(args.THRESHOLD), "threshold")
-		params.Add(jsonutils.NewString(args.COMPARATOR), "comparator")
-		params.Add(jsonutils.NewString(args.RECIPIENTS), "recipients")
-		params.Add(jsonutils.NewString(args.LEVEL), "level")
-		params.Add(jsonutils.NewString(args.CHANNEL), "channel")
-		if len(args.Provider) > 0 {
-			params.Add(jsonutils.NewString(args.Provider), "provider")
-		}
-		if len(args.AccountId) > 0 {
-			params.Add(jsonutils.NewString(args.AccountId), "account_id")
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
 		}
 
 		rst, err := modules.MeterAlert.Create(s, params)
-
 		if err != nil {
 			return err
 		}
@@ -103,27 +94,15 @@ func init() {
 		Type          string `help:"Alarm rule type" choices:"balance|resFee|monthFee"`
 		CloudProvider string `help:"Name of cloud provider, case sensitive"`
 		AccountId     string `help:"Cloud account ID"`
+		ProjectId     string `help:"ID of the project" required:"true"`
 		options.BaseListOptions
 	}
 	R(&MeterAlertListOptions{}, "meteralert-list", "List meter alert", func(s *mcclient.ClientSession, args *MeterAlertListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-			if len(args.Type) > 0 {
-				params.Add(jsonutils.NewString(args.Type), "type")
-			}
-			if len(args.CloudProvider) > 0 {
-				params.Add(jsonutils.NewString(args.CloudProvider), "provider")
-			}
-			if len(args.AccountId) > 0 {
-				params.Add(jsonutils.NewString(args.AccountId), "account_id")
-			}
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
+
 		result, err := modules.MeterAlert.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/networks.go

@@ -34,6 +34,8 @@ func init() {
 		Host       string `help:"search networks attached to a host"`
 		Vpc        string `help:"search networks belongs to a VPC"`
 		Region     string `help:"search networks belongs to a CloudRegion" json:"cloudregion"`
+		City       string `help:"search networks belongs to a city"`
+		Usable     *bool  `help:"search usable networks"`
 		ServerType string `help:"search networks belongs to a ServerType" choices:"guest|baremetal|container|pxe|ipmi"`
 	}
 	R(&NetworkListOptions{}, "network-list", "List networks", func(s *mcclient.ClientSession, opts *NetworkListOptions) error {

cmd/climc/shell/routetables.go

@@ -39,10 +39,10 @@ func init() {
 				typ, _ := routeObj.GetString("type")
 				cidr, _ := routeObj.GetString("cidr")
 				next_hop_type, _ := routeObj.GetString("next_hop_type")
-				next_hop, _ := routeObj.GetString("next_hop")
+				next_hop_id, _ := routeObj.GetString("next_hop_id")
 				route := fmt.Sprintf("%8s: %18s %s", typ, cidr, next_hop_type)
-				if len(next_hop) > 0 {
-					route += fmt.Sprintf(":%s", next_hop)
+				if next_hop_id != "" {
+					route += fmt.Sprintf(":%s", next_hop_id)
 				}
 				routes = append(routes, route)
 			}

cmd/climc/shell/schedulers.go

@@ -107,6 +107,19 @@ func init() {
 			return nil
 		})
 
+	type SchedulerCleanCacheOptions struct {
+		HostId    string `help:"ID of host" short-token:"h"`
+		SessionId string `help:"Session id" short-token:"s"`
+	}
+	R(&SchedulerCleanCacheOptions{}, "scheduler-clean-cache", "Clean scheduler hosts cache",
+		func(s *mcclient.ClientSession, args *SchedulerCleanCacheOptions) error {
+			err := modules.SchedManager.CleanCache(s, args.HostId, args.SessionId)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
 	type SchedulerHistoryListOptions struct {
 		Limit  int  `default:"50" help:"Page limit"`
 		Offset int  `default:"0" help:"Page offset"`

cmd/climc/shell/servers.go

@@ -18,9 +18,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 
+	yaml "gopkg.in/yaml.v2"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
@@ -101,6 +103,32 @@ func init() {
 		return nil
 	})
 
+	R(&options.ServerBatchMetadataOptions{}, "server-batch-add-tag", "add tags for some server", func(s *mcclient.ClientSession, opts *options.ServerBatchMetadataOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Servers.PerformClassAction(s, "batch-user-metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.ServerBatchMetadataOptions{}, "server-batch-set-tag", "Set tags for some server", func(s *mcclient.ClientSession, opts *options.ServerBatchMetadataOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Servers.PerformClassAction(s, "batch-set-user-metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.ServerMetadataOptions{}, "server-add-tag", "Set tag of a server", func(s *mcclient.ClientSession, opts *options.ServerMetadataOptions) error {
 		params, err := opts.Params()
 		if err != nil {
@@ -818,34 +846,101 @@ func init() {
 	})
 
 	type ServersImportFromLibvirtOptions struct {
-		CONFIG_FILE string `help:"JSON file describing servers from libvirt, e.g. 
-			{'hosts': 
-				[
-					{
-						'servers': [
-							{
-								'mac_ip': {'mac1': 'ip1', ...}
-							},
-						],
-						'xml_file_path': '/etc/libvirt/qemu',
-						'host_ip': '192.168.1.100',
-					},
-				],
-			}
-		"`
+		CONFIG_FILE string `help:"File Path describing servers from libvirt"`
 	}
+
+	type Servers struct {
+		Mac string `yaml:"mac"`
+		Ip  string `yaml:"ip"`
+	}
+
+	type Hosts struct {
+		HostIp      string    `yaml:"host_ip"`
+		XmlFilePath string    `yaml:"xml_file_path"`
+		MonitorPath string    `yaml:"monitor_path"`
+		Servers     []Servers `yaml:"servers"`
+	}
+
+	type LibvirtImportOptions struct {
+		Hosts []Hosts `yaml:"hosts"`
+	}
+
 	R(&ServersImportFromLibvirtOptions{}, "servers-import-from-libvirt", "Import servers from libvrt", func(s *mcclient.ClientSession, args *ServersImportFromLibvirtOptions) error {
-		rawConfig, err := ioutil.ReadFile(args.CONFIG_FILE)
+		var (
+			rawConfig []byte
+			err       error
+		)
+
+		rawConfig, err = ioutil.ReadFile(args.CONFIG_FILE)
 		if err != nil {
-			return fmt.Errorf("Read file error: %s", err)
+			return fmt.Errorf("Read config file %s error: %s", args.CONFIG_FILE, err)
 		}
 
-		config := &compute.SLibvirtImportConfig{}
-		err = json.Unmarshal(rawConfig, config)
-		if err != nil {
-			return fmt.Errorf("Parse config error %s", err)
+		var (
+			params []jsonutils.JSONObject
+			config = &compute.SLibvirtImportConfig{}
+		)
+
+		// Try parse as json first
+		{
+			err = json.Unmarshal(rawConfig, config)
+			if err != nil {
+				goto YAML
+			}
+			for i := 0; i < len(config.Hosts); i++ {
+				if nIp := net.ParseIP(config.Hosts[i].HostIp); nIp == nil {
+					return fmt.Errorf("Parse host ip %s failed", config.Hosts[i].HostIp)
+				}
+				for _, server := range config.Hosts[i].Servers {
+					for mac, ip := range server.MacIp {
+						if _, err := net.ParseMAC(mac); err != nil {
+							return fmt.Errorf("Parse mac %s error %s", mac, err)
+						}
+						if nIp := net.ParseIP(ip); nIp == nil {
+							return fmt.Errorf("Parse ip %s failed", ip)
+						}
+					}
+				}
+			}
+
+			goto REQUEST
 		}
-		params, err := jsonutils.Marshal(config.Hosts).GetArray()
+
+	YAML: // Try Parse as yaml
+		{
+			yamlConfig := &LibvirtImportOptions{}
+			err = yaml.Unmarshal(rawConfig, yamlConfig)
+			if err != nil {
+				return err
+			}
+
+			config.Hosts = make([]compute.SLibvirtHostConfig, len(yamlConfig.Hosts))
+			for i := 0; i < len(yamlConfig.Hosts); i++ {
+				if nIp := net.ParseIP(yamlConfig.Hosts[i].HostIp); nIp == nil {
+					return fmt.Errorf("Parse host ip %s failed", yamlConfig.Hosts[i].HostIp)
+				}
+				config.Hosts[i].HostIp = yamlConfig.Hosts[i].HostIp
+				config.Hosts[i].XmlFilePath = yamlConfig.Hosts[i].XmlFilePath
+				config.Hosts[i].Servers = make([]compute.SLibvirtServerConfig, len(yamlConfig.Hosts[i].Servers))
+				for j := 0; j < len(yamlConfig.Hosts[i].Servers); j++ {
+					config.Hosts[i].Servers[j].MacIp = make(map[string]string)
+					mac := yamlConfig.Hosts[i].Servers[j].Mac
+					_, err := net.ParseMAC(mac)
+					if err != nil {
+						return fmt.Errorf("Parse mac address %s error %s", mac, err)
+					}
+					ip := yamlConfig.Hosts[i].Servers[j].Ip
+					nIp := net.ParseIP(ip)
+					if len(nIp) == 0 {
+						return fmt.Errorf("Parse ip address %s failed", ip)
+					}
+					config.Hosts[i].Servers[j].MacIp[mac] = ip
+				}
+			}
+		}
+
+	REQUEST:
+		params, err = jsonutils.Marshal(config.Hosts).GetArray()
 		if err != nil {
 			return err
 		}

cmd/climc/shell/snapshots.go

@@ -93,4 +93,20 @@ func init() {
 		return nil
 	})
 
+	type SnapshotCreateOptions struct {
+		Disk string `help:"Id of disk to take snapshot" json:"disk" required:"true"`
+		NAME string `help:"Name of snapshot" json:"name"`
+	}
+	R(&SnapshotCreateOptions{}, "snapshot-create", "Create a snapshot", func(s *mcclient.ClientSession, args *SnapshotCreateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		snapshot, err := modules.Snapshots.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(snapshot)
+		return nil
+	})
 }

cmd/climc/shell/storages.go

@@ -94,7 +94,7 @@ func init() {
 		ZONE         string `help:"Zone id of storage"`
 		Capacity     int64  `help:"Capacity of the Storage"`
 		MediumType   string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"`
-		StorageType  string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|baremetal"`
+		StorageType  string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|gpfs|baremetal"`
 		MonHost      string `help:"Ceph mon_host config"`
 		Key          string `help:"Ceph key config"`
 		Pool         string `help:"Ceph Poll Name"`

cmd/cloudir/main.go

@@ -14,8 +14,13 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/cloudir/service"
+import (
+	"yunion.io/x/onecloud/pkg/cloudir/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/cloutpost/main.go

@@ -14,8 +14,13 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/cloutpost/service"
+import (
+	"yunion.io/x/onecloud/pkg/cloutpost/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/esxicli/main.go

@@ -88,7 +88,7 @@ func newClient(options *BaseOptions) (*esxi.SESXiClient, error) {
 		return nil, fmt.Errorf("Missing password")
 	}
 
-	return esxi.NewESXiClient("", "", options.Host, options.Port, options.Account, options.Password)
+	return esxi.NewESXiClient2("", "", options.Host, options.Port, options.Account, options.Password, false)
 }
 
 func main() {

cmd/glance/main.go

@@ -16,8 +16,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/image/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/host-image/main.go

@@ -14,8 +14,13 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/hostimage"
+import (
+	"yunion.io/x/onecloud/pkg/hostimage"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	hostimage.StartService()
 }

cmd/host/main.go

@@ -14,9 +14,18 @@
 
 package main
 
-import "yunion.io/x/onecloud/pkg/hostman"
+import (
+	"yunion.io/x/onecloud/pkg/cloudcommon/service"
+	"yunion.io/x/onecloud/pkg/hostman"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
-	var srv = hostman.SHostService{}
+	defer atexit.Handle()
+
+	var srv = &hostman.SHostService{}
+	srv.SServiceBase = &service.SServiceBase{
+		Service: srv,
+	}
 	srv.StartService()
 }

cmd/lbagent/main.go

@@ -26,9 +26,12 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/lbagent"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	opts := &lbagent.Options{}
 	commonOpts := &opts.CommonOptions
 	{

cmd/logger/main.go

@@ -16,8 +16,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/logger/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/openstackcli/main.go

@@ -34,6 +34,7 @@ type BaseOptions struct {
 	Password     string `help:"Password" default:"$OPENSTACK_PASSWORD"`
 	Project      string `help:"Project" default:"$OPENSTACK_PROJECT"`
 	EndpointType string `help:"Project" default:"$OPENSTACK_ENDPOINT_TYPE|internal"`
+	DomainName   string `help:"DomainName" default:"$OPENSTACK_DOMAIN_NAME"`
 	RegionID     string `help:"RegionId" default:"$OPENSTACK_REGION_ID"`
 	SUBCOMMAND   string `help:"openstackcli subcommand" subcommand:"true"`
 }
@@ -91,7 +92,7 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
-	cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.Debug)
+	cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.DomainName, options.Debug)
 	if err != nil {
 		return nil, err
 	}

cmd/region-dns/main.go

@@ -34,6 +34,7 @@ import (
 	_ "github.com/mholt/caddy/startupshutdown"
 
 	_ "yunion.io/x/onecloud/pkg/dns"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 var directives = []string{
@@ -58,5 +59,7 @@ func init() {
 }
 
 func main() {
+	defer atexit.Handle()
+
 	coremain.Run()
 }

cmd/region/main.go

@@ -16,8 +16,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/compute/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/scheduler/main.go

@@ -15,17 +15,20 @@
 package main
 
 import (
-	"os"
-
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/scheduler/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	exitCode := 0
+	defer func() {
+		atexit.Exit(exitCode)
+	}()
+
 	if err := service.StartService(); err != nil {
 		log.Errorln(err)
-		os.Exit(-1)
+		exitCode = -1
 	}
-	os.Exit(0)
 }

cmd/torrent/main.go

@@ -33,6 +33,7 @@ import (
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/nodeid"
 	"yunion.io/x/onecloud/pkg/util/torrentutils"
@@ -62,6 +63,8 @@ func exitSignalHandlers(client *torrent.Client) {
 }
 
 func main() {
+	defer atexit.Handle()
+
 	options := Options{}
 
 	parser, err := structarg.NewArgumentParser(&options, "torrent-srv", "bit-torrent server", "2018")

cmd/ucloudcli/main.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"yunion.io/x/onecloud/pkg/util/ucloud"
+
+	"yunion.io/x/log"
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	_ "yunion.io/x/onecloud/pkg/util/ucloud/shell"
+)
+
+type BaseOptions struct {
+	Help  bool `help:"Show help" default:"false"`
+	Debug bool `help:"Show debug" default:"false"`
+
+	AccessKey  string `help:"Access key" default:"$UCLOUD_ACCESS_KEY"`
+	Secret     string `help:"Secret" default:"$UCLOUD_SECRET"`
+	RegionId   string `help:"RegionId" default:"$UCLOUD_REGION"`
+	ProjectId  string `help:"ProjectId" default:"$UCLOUD_PROJECT"`
+	SUBCOMMAND string `help:"ucloudcli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"ucloudcli",
+		"Command-line interface to ucloud API.",
+		`See "ucloudcli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	log.Errorf("%s", e)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*ucloud.SRegion, error) {
+	if len(options.AccessKey) == 0 {
+		return nil, fmt.Errorf("Missing accessKey")
+	}
+
+	if len(options.Secret) == 0 {
+		return nil, fmt.Errorf("Missing secret")
+	}
+
+	account := ""
+	if len(options.ProjectId) > 0 {
+		account = options.AccessKey + "::" + options.ProjectId
+	} else {
+		account = options.AccessKey
+	}
+
+	cli, err := ucloud.NewUcloudClient("", "", account, options.Secret, options.Debug)
+	if err != nil {
+		return nil, err
+	}
+
+	region := cli.GetRegion(options.RegionId)
+	if region == nil {
+		return nil, fmt.Errorf("No such region %s", options.RegionId)
+	}
+
+	return region, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var region *ucloud.SRegion
+				region, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(region, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/webconsole/main.go

@@ -15,9 +15,12 @@
 package main
 
 import (
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/yunionconf/main.go

@@ -20,9 +20,12 @@ Yunion Conf Service
 */
 
 import (
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/yunionconf/service"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

pkg/apis/compute/api.go

@@ -25,6 +25,7 @@ type SchedtagConfig struct {
 
 	Id       string `json:"id"`
 	Strategy string `json:"strategy"`
+	Weight   int    `json:"weight"`
 }
 
 type NetworkConfig struct {

pkg/apis/compute/baremetal.go

@@ -0,0 +1,11 @@
+package compute
+
+type SGMapItem struct {
+	SGDeviceName    string
+	HostNumber      int
+	Bus             int
+	SCSIId          int
+	Lun             int
+	Type            int
+	LinuxDeviceName string
+}

pkg/apis/compute/cloudaccount_const.go

@@ -12,6 +12,7 @@ const (
 	CLOUD_PROVIDER_SYNC_STATUS_QUEUED  = "queued"
 	CLOUD_PROVIDER_SYNC_STATUS_SYNCING = "syncing"
 	CLOUD_PROVIDER_SYNC_STATUS_IDLE    = "idle"
+	CLOUD_PROVIDER_SYNC_STATUS_ERROR   = "error"
 
 	CLOUD_PROVIDER_ONECLOUD  = "OneCloud"
 	CLOUD_PROVIDER_VMWARE    = "VMware"
@@ -23,15 +24,17 @@ const (
 	CLOUD_PROVIDER_OPENSTACK = "OpenStack"
 	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
 
-	CLOUD_PROVIDER_HEALTH_NORMAL       = "normal"       // 远端处于健康状态
-	CLOUD_PROVIDER_HEALTH_INSUFFICIENT = "insufficient" // 不足按需资源余额
-	CLOUD_PROVIDER_HEALTH_SUSPENDED    = "suspended"    // 远端处于冻结状态
-	CLOUD_PROVIDER_HEALTH_ARREARS      = "arrears"      // 远端处于欠费状态
-	CLOUD_PROVIDER_HEALTH_UNKNOWN      = "unknown"      // 未知状态，查询失败
+	CLOUD_PROVIDER_HEALTH_NORMAL        = "normal"        // 远端处于健康状态
+	CLOUD_PROVIDER_HEALTH_INSUFFICIENT  = "insufficient"  // 不足按需资源余额
+	CLOUD_PROVIDER_HEALTH_SUSPENDED     = "suspended"     // 远端处于冻结状态
+	CLOUD_PROVIDER_HEALTH_ARREARS       = "arrears"       // 远端处于欠费状态
+	CLOUD_PROVIDER_HEALTH_UNKNOWN       = "unknown"       // 未知状态，查询失败
+	CLOUD_PROVIDER_HEALTH_NO_PERMISSION = "no permission" // 没有权限获取账单信息
 )
 
 var (
-	CLOUD_PROVIDER_VALID_STATUS = []string{CLOUD_PROVIDER_CONNECTED}
+	CLOUD_PROVIDER_VALID_STATUS        = []string{CLOUD_PROVIDER_CONNECTED}
+	CLOUD_PROVIDER_VALID_HEALTH_STATUS = []string{CLOUD_PROVIDER_HEALTH_NORMAL, CLOUD_PROVIDER_HEALTH_NO_PERMISSION}
 
 	CLOUD_PROVIDERS = []string{
 		CLOUD_PROVIDER_ONECLOUD,

pkg/apis/compute/cloudregions_const.go

@@ -5,4 +5,138 @@ const (
 	CLOUD_REGION_STATUS_OUTOFSERVICE = "outofservice"
 
 	DEFAULT_REGION_ID = "default"
+
+	// 中国
+	CITY_QING_DAO      = "Qingdao"     //青岛
+	CITY_BEI_JING      = "Beijing"     //北京
+	CITY_ZHANG_JIA_KOU = "Zhangjiakou" //张家口
+	CITY_HU_HE_HAO_TE  = "Huhehaote"   //呼和浩特
+	CITY_HANG_ZHOU     = "Hangzhou"    //杭州
+	CITY_SHANG_HAI     = "Shanghai"    //上海
+	CITY_SHEN_ZHEN     = "Shenzhen"    //深圳
+	CITY_HONG_KONG     = "Hongkong"    //香港
+	CITY_NING_XIA      = "Ningxia"     //宁夏
+	CITY_GUANG_ZHOU    = "Guangzhou"   //广州
+	CITY_GUI_YANG      = "Guiyang"     //贵阳
+	CITY_TAIPEI        = "Taipei"      //台北市
+	CITY_KAOHSIUNG     = "Kaohsiung"   //高雄市
+	CITY_CHENG_DU      = "Chengdu"     //成都
+	CITY_CHONG_QING    = "Chongqing"   //重庆
+
+	// 日本
+	CITY_TOKYO = "Tokyo" //东京
+	CITY_OSAKA = "Osaka" //大阪市
+
+	// 新加坡
+	CITY_SINGAPORE = "Singapore" //新加坡
+
+	// 澳大利亚
+	CITY_SYDNEY     = "Sydney"     //悉尼
+	CITY_YARRALUMLA = "Yarralumla" //亚拉伦拉
+	CITY_MELBOURNE  = "Melbourne"  //墨尔本
+
+	// 马来西亚
+	CITY_KUALA_LUMPUR = "Kuala Lumpur" //吉隆坡
+
+	// 印度尼西亚
+	CITY_JAKARTA = "Jakarta" //雅加达
+
+	// 印度
+	CITY_MUMBAI      = "Mumbai"      //孟买
+	CITY_KANCHIPURAM = "Kanchipuram" //甘吉布勒姆
+	CITY_MAHARASHTRA = "Maharashtra" //马哈拉施特拉邦
+
+	// 美国
+	CITY_VIRGINIA      = "Virginia"      //弗吉尼亚
+	CITY_SILICONVALLEY = "Siliconvalley" //硅谷
+	CITY_OHIO          = "Ohio"          //俄亥俄州
+	CITY_N_VIRGINIA    = "N. Virginia"   //北弗吉尼亚
+	CITY_N_CALIFORNIA  = "N. California" //北加州
+	CITY_OREGON        = "Oregon"        //俄勒冈州
+	CITY_LOS_ANGELES   = "Los Angeles"   //洛杉矶
+	CITY_SAN_FRANCISCO = "San Francisco" //旧金山
+	CITY_UTAH          = "Utah"          //犹他州
+	CITY_WASHINGTON    = "Washington"    //华盛顿
+	CITY_TEXAS         = "Texas"         //德克萨斯
+	CITY_CHICAGO       = "Chicago"       //芝加哥
+	CITY_IOWA          = "Iowa"          //爱荷华
+	CITY_US_GOV_WEST   = "us-gov-west"   //???
+
+	// 英国
+	CITY_LONDON      = "London"      //伦敦
+	CITY_HALTON      = "Halton"      //哈尔顿
+	CITY_WEST_SUSSEX = "West Sussex" //西苏塞克斯
+
+	// 阿拉伯联合酋长国
+	CITY_DUBAI = "Dubai" //迪拜
+
+	// 德国
+	CITY_FRANKFURT = "Frankfurt" //法兰克福
+
+	// 韩国
+	CITY_SEOUL = "Seoul" //首尔
+	CITY_BUSAN = "Busan" //釜山
+
+	// 加拿大
+	CITY_CANADA_CENTRAL = "Canada Central" //加拿大中部
+	CITY_QUEBEC         = "Quebec"         //魁北克市
+	CITY_TORONTO        = "Toronto"        //多伦多
+
+	// 爱尔兰
+	CITY_IRELAND = "Ireland" //爱尔兰
+	CITY_DUBLIN  = "Dublin"  //都柏林
+
+	// 法国
+	CITY_PARIS  = "Paris"  //巴黎
+	CITY_ALLIER = "Allier" //阿利埃河
+	CITY_TARN   = "Tarn"   //塔恩
+
+	// 瑞典
+	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
+
+	// 巴西
+	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+
+	// 荷兰
+	CITY_HOLLAND = "Holland" //荷兰
+
+	// 南非
+	CITY_PRETORIA  = "Pretoria"  //比勒陀利亚
+	CITY_CAPE_TOWN = "Cape Town" //开普敦
+
+	// 泰国
+	CITY_BANGKOK = "Bangkok" //曼谷
+
+	// 俄罗斯
+	CITY_MOSCOW = "Moscow" //莫斯科
+
+	// 尼日利亚
+	CITY_LAGOS = "Lagos" //拉哥斯
+
+	// 越南
+	CITY_HO_CHI_MINH = "Ho Chi Minh" //???
+
+	COUNTRY_CODE_CN = "CN" //中国
+	COUNTRY_CODE_JP = "JP" //日本
+	COUNTRY_CODE_SG = "SG" //新加坡
+	COUNTRY_CODE_AU = "AU" //澳大利亚
+	COUNTRY_CODE_MY = "MY" //马来西亚
+	COUNTRY_CODE_ID = "ID" //印度尼西亚
+	COUNTRY_CODE_IN = "IN" //印度
+	COUNTRY_CODE_US = "US" //美国
+	COUNTRY_CODE_GB = "GB" //英国
+	COUNTRY_CODE_AE = "AE" //阿拉伯联合酋长国
+	COUNTRY_CODE_DE = "DE" //德国
+	COUNTRY_CODE_KR = "KR" //韩国
+	COUNTRY_CODE_CA = "CA" //加拿大
+	COUNTRY_CODE_IE = "IE" //爱尔兰
+	COUNTRY_CODE_FR = "FR" //法国
+	COUNTRY_CODE_SE = "SE" //瑞典
+	COUNTRY_CODE_BR = "BR" //巴西
+	COUNTRY_CODE_NL = "NL" //荷兰
+	COUNTRY_CODE_ZA = "ZA" //南非
+	COUNTRY_CODE_TH = "TH" //泰国
+	COUNTRY_CODE_RU = "RU" //俄罗斯
+	COUNTRY_CODE_NG = "NG" //尼日利亚
+	COUNTRY_CODE_VN = "VN" //越南
 )

pkg/apis/compute/disk_const.go

@@ -51,4 +51,6 @@ const (
 	DISK_TYPE_VOLUME = "volume"
 
 	DISK_BACKING_IMAGE = "image"
+
+	DISK_SIZE_AUTOEXTEND = -1
 )

pkg/apis/compute/esxi_const.go

@@ -0,0 +1,5 @@
+package compute
+
+const (
+	ESXI_IMAGE_CACHE_TMP_PREFIX = "yunioncloud.imagecache"
+)

pkg/apis/compute/guest_const.go

@@ -72,13 +72,13 @@ const (
 	VM_REBUILD_ROOT       = "rebuild_root"
 	VM_REBUILD_ROOT_FAIL  = "rebuild_root_fail"
 
-	VM_START_SNAPSHOT  = "snapshot_start"
-	VM_SNAPSHOT        = "snapshot"
-	VM_SNAPSHOT_DELETE = "snapshot_delete"
-	VM_BLOCK_STREAM    = "block_stream"
-	VM_MIRROR_FAIL     = "mirror_failed"
-	VM_SNAPSHOT_SUCC   = "snapshot_succ"
-	VM_SNAPSHOT_FAILED = "snapshot_failed"
+	VM_START_SNAPSHOT    = "snapshot_start"
+	VM_SNAPSHOT          = "snapshot"
+	VM_SNAPSHOT_DELETE   = "snapshot_delete"
+	VM_BLOCK_STREAM      = "block_stream"
+	VM_BLOCK_STREAM_FAIL = "block_stream_fail"
+	VM_SNAPSHOT_SUCC     = "snapshot_succ"
+	VM_SNAPSHOT_FAILED   = "snapshot_failed"
 
 	VM_SYNCING_STATUS = "syncing"
 	VM_SYNC_CONFIG    = "sync_config"
@@ -186,7 +186,8 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 }
 
 const (
-	VM_AWS_DEFAULT_LOGIN_USER = "ec2user"
+	VM_AWS_DEFAULT_LOGIN_USER   = "ec2user"
+	VM_AZURE_DEFAULT_LOGIN_USER = "toor"
 
 	VM_METADATA_APP_TAGS            = "app_tags"
 	VM_METADATA_CREATE_PARAMS       = "create_params"

pkg/apis/compute/guest_import_api.go

@@ -53,6 +53,7 @@ type SImportGuestDesc struct {
 	BootOrder   string            `json:"boot_order"`
 	IsSystem    bool              `json:"is_system"`
 	Description string            `json:"description"`
+	MonitorPath string            `json:"monitor_path"`
 }
 
 type SLibvirtServerConfig struct {
@@ -62,6 +63,7 @@ type SLibvirtServerConfig struct {
 type SLibvirtHostConfig struct {
 	Servers     []SLibvirtServerConfig `json:"servers"`
 	XmlFilePath string                 `json:"xml_file_path"`
+	MonitorPath string                 `json:"monitor_path"`
 	HostIp      string                 `json:"host_ip"`
 }
 

pkg/apis/compute/host_const.go

@@ -74,6 +74,6 @@ const (
 	HostResourceTypeDedicated      = "dedicated"
 )
 
-var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN, HOST_TYPE_AZURE, HOST_TYPE_AWS, HOST_TYPE_QCLOUD, HOST_TYPE_HUAWEI, HOST_TYPE_OPENSTACK}
+var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN, HOST_TYPE_AZURE, HOST_TYPE_AWS, HOST_TYPE_QCLOUD, HOST_TYPE_HUAWEI, HOST_TYPE_OPENSTACK, HOST_TYPE_UCLOUD}
 
 var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN}

pkg/apis/compute/loadbalancer_const.go

@@ -81,6 +81,7 @@ var LB_BACKENDGROUP_TYPE = choices.NewChoices(
 )
 
 const (
+	LB_ALIYUN_SPEC_SHAREABLE = "" //性能共享型
 	LB_ALIYUN_SPEC_S1_SMALL  = "slb.s1.small"
 	LB_ALIYUN_SPEC_S2_SMALL  = "slb.s2.small"
 	LB_ALIYUN_SPEC_S3_SMALL  = "slb.s3.small"
@@ -95,6 +96,7 @@ const (
 )
 
 var LB_ALIYUN_SPECS = choices.NewChoices(
+	LB_ALIYUN_SPEC_SHAREABLE,
 	LB_ALIYUN_SPEC_S1_SMALL,
 	LB_ALIYUN_SPEC_S2_SMALL,
 	LB_ALIYUN_SPEC_S3_SMALL,
@@ -254,6 +256,7 @@ const (
 	LB_SCHEDULER_WLC = "wlc" // weighted least connection
 	LB_SCHEDULER_SCH = "sch" // source-ip-based consistent hash
 	LB_SCHEDULER_TCH = "tch" // 4-tuple-based consistent hash
+	LB_SCHEDULER_QCH = "qch"
 )
 
 var LB_SCHEDULER_TYPES = choices.NewChoices(
@@ -264,15 +267,32 @@ var LB_SCHEDULER_TYPES = choices.NewChoices(
 	LB_SCHEDULER_TCH,
 )
 
+var LB_ALIYUN_UDP_SCHEDULER_TYPES = choices.NewChoices(
+	LB_SCHEDULER_RR,
+	LB_SCHEDULER_WRR,
+	LB_SCHEDULER_WLC,
+	LB_SCHEDULER_SCH,
+	LB_SCHEDULER_TCH,
+	LB_SCHEDULER_QCH,
+)
+
+var LB_ALIYUN_COMMON_SCHEDULER_TYPES = choices.NewChoices(
+	LB_SCHEDULER_RR,
+	LB_SCHEDULER_WRR,
+	LB_SCHEDULER_WLC,
+)
+
 // TODO raw type
 const (
 	LB_BACKEND_GUEST = "guest"
 	LB_BACKEND_HOST  = "host"
+	LB_BACKEND_IP    = "ip"
 )
 
 var LB_BACKEND_TYPES = choices.NewChoices(
 	LB_BACKEND_GUEST,
 	LB_BACKEND_HOST,
+	LB_BACKEND_IP,
 )
 
 const (

pkg/apis/compute/network_const.go

@@ -49,4 +49,9 @@ var (
 		NETWORK_TYPE_PXE,
 		NETWORK_TYPE_IPMI,
 	}
+
+	REGINAL_NETWORK_PROVIDERS = []string{
+		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_UCLOUD,
+	}
 )

pkg/apis/compute/storage_const.go

@@ -23,6 +23,7 @@ const (
 	STORAGE_NAS       = "nas"
 	STORAGE_VSAN      = "vsan"
 	STORAGE_NFS       = "nfs"
+	STORAGE_GPFS      = "gpfs"
 
 	STORAGE_PUBLIC_CLOUD     = "cloud"
 	STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency"
@@ -83,7 +84,7 @@ var (
 	STORAGE_ALL_TYPES     = []string{
 		STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN,
-		STORAGE_NFS,
+		STORAGE_NFS, STORAGE_GPFS,
 	}
 	STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS,
@@ -94,7 +95,15 @@ var (
 		STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA,
 		STORAGE_OPENSTACK_ISCSI, STORAGE_UCLOUD_CLOUD_NORMAL, STORAGE_UCLOUD_CLOUD_SSD,
 		STORAGE_UCLOUD_LOCAL_NORMAL, STORAGE_UCLOUD_LOCAL_SSD, STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK,
+		STORAGE_GPFS,
 	}
 
-	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL}
+
+	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
+
+	SHARED_FILE_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS}
+
+	// 目前来说只支持这些
+	SHARED_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS, STORAGE_RBD}
 )

pkg/apis/image/doc.go

@@ -0,0 +1 @@
+package image // import "yunion.io/x/onecloud/pkg/apis/image"

pkg/apis/scheduler/api.go

@@ -52,9 +52,6 @@ type ServerConfig struct {
 	Name        string `json:"name"`
 	GuestStatus string `json:"guest_status"`
 
-	// HostId used by migrate
-	HostId string `json:"host_id"`
-
 	// DEPRECATED
 	Metadata       map[string]string `json:"__meta__"`
 	ForGuests      []*ForGuest       `json:"for_guests"`
@@ -70,6 +67,12 @@ type ScheduleInput struct {
 	ScheduleBaseConfig
 
 	ServerConfig
+
+	// HostId used by migrate
+	HostId       string `json:"host_id"`
+	LiveMigrate  bool   `json:"live_migrate"`
+	CpuDesc      string `json:"cpu_desc"`
+	CpuMicrocode string `json:"cpu_microcode"`
 }
 
 func (input ScheduleInput) ToConditionInput() *jsonutils.JSONDict {
@@ -90,10 +93,11 @@ type CandidateNet struct {
 }
 
 type CandidateResource struct {
-	HostId string           `json:"host_id"`
-	Name   string           `json:"name"`
-	Disks  []*CandidateDisk `json:"disks"`
-	Nets   []*CandidateNet  `json:"nets"`
+	SessionId string           `json:"session_id"`
+	HostId    string           `json:"host_id"`
+	Name      string           `json:"name"`
+	Disks     []*CandidateDisk `json:"disks"`
+	Nets      []*CandidateNet  `json:"nets"`
 
 	// used by backup schedule
 	BackupCandidate *CandidateResource `json:"backup_candidate"`

pkg/appsrv/appsrv.go

@@ -245,7 +245,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 		if ok {
 			fw := newResponseWriterChannel(w)
 			worker := make(chan *SWorker)
-			to := hand.processTimeout
+			to := hand.FetchProcessTimeout(r)
 			if to == 0 {
 				to = app.processTimeout
 			}

pkg/appsrv/handlerinfo.go

@@ -41,6 +41,14 @@ type SHandlerInfo struct {
 	skipLog        bool
 }
 
+func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration {
+	if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
+		return time.Hour * 2
+	} else {
+		return this.processTimeout
+	}
+}
+
 func (this *SHandlerInfo) GetName(params map[string]string) string {
 	if len(this.name) > 0 {
 		return this.name

pkg/baremetal/manager.go

@@ -25,9 +25,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/util/errors"
+	yerrors "yunion.io/x/pkg/util/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/seclib"
 	"yunion.io/x/pkg/util/sets"
@@ -120,7 +122,7 @@ func (m *SBaremetalManager) loadConfigs() error {
 			errs = append(errs, <-errsChannel)
 		}
 	}
-	return errors.NewAggregate(errs)
+	return yerrors.NewAggregate(errs)
 }
 
 func (m *SBaremetalManager) initBaremetal(session *mcclient.ClientSession, bmId string) error {
@@ -191,7 +193,7 @@ func (m *SBaremetalManager) GetBaremetalById(bmId string) *SBaremetalInstance {
 }
 
 func (m *SBaremetalManager) GetBaremetalByMac(mac net.HardwareAddr) pxe.IBaremetalInstance {
-	var obj *SBaremetalInstance
+	var obj pxe.IBaremetalInstance
 	getter := func(key, val interface{}) bool {
 		instance := val.(*SBaremetalInstance)
 		if instance.GetNicByMac(mac) != nil {
@@ -282,7 +284,7 @@ func (b *SBaremetalInstance) GetClientSession() *mcclient.ClientSession {
 func (b *SBaremetalInstance) GetId() string {
 	id, err := b.desc.GetString("id")
 	if err != nil {
-		log.Fatalf("Get id from desc error: %v", err)
+		log.Errorf("Get id from desc %s error: %v", b.desc.String(), err)
 	}
 	return id
 }
@@ -290,7 +292,7 @@ func (b *SBaremetalInstance) GetId() string {
 func (b *SBaremetalInstance) GetName() string {
 	id, err := b.desc.GetString("name")
 	if err != nil {
-		log.Fatalf("Get name from desc error: %v", err)
+		log.Fatalf("Get name from desc %s error: %v", b.desc.String(), err)
 	}
 	return id
 }
@@ -690,24 +692,41 @@ func (b *SBaremetalInstance) GetNotifyUrl() string {
 	return fmt.Sprintf("%s/baremetals/%s/notify", b.manager.Agent.GetManagerUri(), b.GetId())
 }
 
+func (b *SBaremetalInstance) getTftpFileUrl(filename string) string {
+	serverIP, err := b.manager.Agent.GetDHCPServerIP()
+	if err != nil {
+		log.Errorf("Get http file server: %v", err)
+		return filename
+	}
+	if o.Options.EnableTftpHttpDownload {
+		return fmt.Sprintf("http://%s:%d/tftp/%s", serverIP, o.Options.Port+1000, filename)
+	}
+	return filename
+}
+
 func (b *SBaremetalInstance) GetTFTPResponse() string {
-	resp := `default start
+	resp := `DEFAULT start
 serial 1 115200
 
-label start
-    menu label ^Start
-    menu default
+LABEL start
+    MENU LABEL ^Start
+    MENU default
 `
 
 	if b.NeedPXEBoot() {
-		resp += "    kernel kernel\n"
-		resp += fmt.Sprintf("    append initrd=initramfs token=%s url=%s",
-			auth.GetTokenString(), b.GetNotifyUrl())
+		resp += fmt.Sprintf("    kernel %s\n", b.getTftpFileUrl("kernel"))
+		args := []string{
+			fmt.Sprintf("initrd=%s", b.getTftpFileUrl("initramfs")),
+			fmt.Sprintf("token=%s", auth.GetTokenString()),
+			fmt.Sprintf("url=%s", b.GetNotifyUrl()),
+		}
+		resp += fmt.Sprintf("    append %s\n", strings.Join(args, " "))
 	} else {
-		resp += "    COM32 chain.c32\n"
+		resp += fmt.Sprintf("    COM32 %s\n", b.getTftpFileUrl("chain.c32"))
 		resp += "    APPEND hd0 0\n"
+		b.ClearSSHConfig()
 	}
-	log.Infof("%s", resp)
+	log.Infof("[TFTP response]: \n%s", resp)
 	return resp
 }
 
@@ -999,6 +1018,7 @@ func (b *SBaremetalInstance) StartNewTask(factory tasks.TaskFactory, taskId stri
 	go func() {
 		task, err := factory(b, taskId, data)
 		if err != nil {
+			log.Errorf("New task %#v error: %v", factory, err)
 			tasks.SetTaskFail(task, err)
 			return
 		}
@@ -1165,7 +1185,7 @@ func (server *SBaremetalServer) GetId() string {
 func (server *SBaremetalServer) GetName() string {
 	id, err := server.desc.GetString("name")
 	if err != nil {
-		log.Fatalf("Get name from desc error: %v", err)
+		log.Errorf("Get name from desc %s error: %v", server.desc.String(), err)
 	}
 	return id
 }
@@ -1311,40 +1331,40 @@ func (s *SBaremetalServer) DoPartitionDisk(term *ssh.Client) ([]*disktool.Partit
 	storages = append(storages, pcie...)
 	confs, err := s.GetDiskConfig()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "do disk config")
 	}
 	layouts, err := baremetal.CalculateLayout(confs, storages)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "CalculateLayout")
 	}
 
 	tool := disktool.NewSSHPartitionTool(term)
 	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
 	err = tool.RetrieveDiskInfo()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "RetrieveDiskInfo")
 	}
 
 	disks, _ := s.desc.GetArray("disks")
 	if len(disks) == 0 {
-		return nil, fmt.Errorf("Empty disks in desc")
+		return nil, errors.New("Empty disks in desc")
 	}
 
 	rootDisk := disks[0]
 	rootSize, _ := rootDisk.Int("size")
 	err = s.doCreateRoot(term, tool.GetRootDisk().GetDevName())
 	if err != nil {
-		return nil, fmt.Errorf("Failed to create root: %v", err)
+		return nil, errors.Wrapf(err, "Failed to create root")
 	}
 
 	tool.RetrievePartitionInfo()
 	parts := tool.GetPartitions()
 	if len(parts) == 0 {
-		return nil, fmt.Errorf("Root disk create failed, no partitions")
+		return nil, errors.New("Root disk create failed, no partitions")
 	}
 	log.Infof("Resize root to %d MB", rootSize)
 	if err := tool.ResizePartition(0, rootSize); err != nil {
-		return nil, fmt.Errorf("Fail to resize root to %d, err: %v", rootSize, err)
+		return nil, errors.Wrapf(err, "Fail to resize root to %d", rootSize)
 	}
 	if len(disks) > 1 {
 		for _, disk := range disks[1:] {
@@ -1357,7 +1377,7 @@ func (s *SBaremetalServer) DoPartitionDisk(term *ssh.Client) ([]*disktool.Partit
 			driver, _ := disk.GetString("driver")
 			log.Infof("Create partition %d %s", sz, fs)
 			if err := tool.CreatePartition(-1, sz, fs, true, driver, uuid); err != nil {
-				return nil, fmt.Errorf("Fail to create disk %s: %v", disk.String(), err)
+				return nil, errors.Wrapf(err, "Fail to create disk %s", disk.String())
 			}
 		}
 	}
@@ -1398,15 +1418,18 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 
 	rootDisk := disks[0]
 	rootSize, _ := rootDisk.Int("size")
-	err = s.doCreateRoot(term, tool.GetRootDisk().GetDevName())
+	rd := tool.GetRootDisk()
+	err = s.doCreateRoot(term, rd.GetDevName())
 	if err != nil {
 		return nil, fmt.Errorf("Failed to create root: %v", err)
 	}
-
 	tool.RetrievePartitionInfo()
+	if err := rd.ReInitInfo(); err != nil {
+		return nil, errors.Wrap(err, "Reinit root disk after create root")
+	}
 
 	log.Infof("Resize root to %d MB", rootSize)
-	if err := tool.ResizePartition(0, rootSize); err != nil {
+	if err := rd.ResizePartition(rootSize); err != nil {
 		return nil, fmt.Errorf("Fail to resize root to %d, err: %v", rootSize, err)
 	}
 	if len(disks) > 1 {
@@ -1417,17 +1440,24 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 			}
 			fs, _ := disk.GetString("fs")
 			uuid, _ := disk.GetString("disk_id")
-			driver, _ := disk.GetString("driver")
 			log.Infof("Create partition %d %s", sz, fs)
-			if err := tool.CreatePartition(0, sz, fs, false, driver, uuid); err != nil {
-				log.Errorf("Rebuild root create (%s, %d, %s, %s) partition error: %v", uuid, sz, fs, driver, err)
+			if err := rd.CreatePartition(sz, fs, false, uuid); err != nil {
+				log.Errorf("Rebuild root create (%s, %d, %s) partition error: %v", uuid, sz, fs, err)
 				break
 			}
 		}
 	}
 	log.Infof("Finish create partitions")
 
-	return tool.GetPartitions(), nil
+	parts := rd.GetPartitions()
+	restDisks := tool.Disks()
+	if len(restDisks) > 1 {
+		restDisks = restDisks[1:]
+	}
+	for _, d := range restDisks {
+		parts = append(parts, d.GetPartitions()...)
+	}
+	return parts, nil
 }
 
 func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {

pkg/baremetal/nic.go

@@ -76,7 +76,11 @@ func GetNicDHCPConfig(
 		case 6:
 			conf.BootFile = "bootia32.efi"
 		default:
-			conf.BootFile = "pxelinux.0"
+			bootFile := "pxelinux.0"
+			if o.Options.EnableTftpHttpDownload {
+				bootFile = "lpxelinux.0"
+			}
+			conf.BootFile = bootFile
 		}
 		pxePath := filepath.Join(o.Options.TftpRoot, conf.BootFile)
 		if f, err := os.Open(pxePath); err != nil {

pkg/baremetal/options/options.go

@@ -22,7 +22,7 @@ type BaremetalOptions struct {
 	ListenInterface        string `help:"Master net interface of baremetal server" default:"br0"`
 	AccessAddress          string `help:"Management IP address of baremetal server, only need to use when multiple address bind to ListenInterface"`
 	ListenAddress          string `help:"PXE serve IP address to select when multiple address bind to ListenInterface" default:"0.0.0.0"`
-	TftpRoot               string `help:"tftp root directory"`
+	TftpRoot               string `default:"/opt/cloud/yunion/baremetal" help:"tftp root directory"`
 	AutoRegisterBaremetal  bool   `default:"true" help:"Automatically create a baremetal instance"`
 	BaremetalsPath         string `default:"/opt/cloud/workspace/baremetals" help:"Path for baremetals configuration files"`
 	LinuxDefaultRootUser   bool   `default:"false" help:"Default account for linux system is root"`
@@ -41,6 +41,7 @@ type BaremetalOptions struct {
 	DefaultStrongIpmiPassword string `help:"Default strong IPMI passowrd"`
 
 	WindowsDefaultAdminUser bool `default:"true" help:"Default account for Windows system is Administrator"`
+	EnableTftpHttpDownload  bool `default:"true" help:"Pxelinux download file through http"`
 }
 
 var (

pkg/baremetal/service/service.go

@@ -15,6 +15,8 @@
 package service
 
 import (
+	"fmt"
+	"net/http"
 	"os"
 
 	"yunion.io/x/log"
@@ -43,12 +45,27 @@ func (s *BaremetalService) StartService() {
 	app := app_common.InitApp(&o.Options.CommonOptions, false)
 	handler.InitHandlers(app)
 
+	s.startFileServer()
+
 	app_common.ServeForeverWithCleanup(app, &o.Options.CommonOptions, func() {
 		tasks.OnStop()
 		baremetal.Stop()
 	})
 }
 
+func (s *BaremetalService) startFileServer() {
+	if !o.Options.EnableTftpHttpDownload {
+		return
+	}
+	fs := http.FileServer(http.Dir(o.Options.TftpRoot))
+	http.Handle("/tftp/", http.StripPrefix("/tftp/", fs))
+	go func() {
+		if err := http.ListenAndServe(fmt.Sprintf("%s:%d", o.Options.Address, o.Options.Port+1000), nil); err != nil {
+			panic(fmt.Sprintf("start http file server: %v", err))
+		}
+	}()
+}
+
 func (s *BaremetalService) startAgent() {
 	err := baremetal.Start()
 	if err != nil {

pkg/baremetal/tasks/base.go

@@ -21,6 +21,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
@@ -287,18 +289,18 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error {
 	log.Infof("EnsurePowerUp: bootdev=pxe")
 	status, err := self.Baremetal.GetPowerStatus()
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "Get power status")
 	}
 	for status == "" || status == types.POWER_STATUS_OFF {
 		if status == types.POWER_STATUS_OFF {
 			err = self.Baremetal.DoPXEBoot()
 			if err != nil {
-				return err
+				return errors.Wrapf(err, "Do PXE boot")
 			}
 		}
 		status, err = self.Baremetal.GetPowerStatus()
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "Get power status")
 		}
 		if status == "" || status == types.POWER_STATUS_OFF {
 			time.Sleep(40 * time.Second)
@@ -367,7 +369,7 @@ func (self *SBaremetalPXEBootTaskBase) InitPXEBootTask(pxeBootTask IPXEBootTask,
 		return self, fmt.Errorf("EnsurePowerShutdown: %v", err)
 	}
 	if err := self.EnsurePowerUp(); err != nil {
-		return self, fmt.Errorf("EnsurePowerUp to pxe: %v", err)
+		return self, errors.Wrapf(err, "EnsurePowerUp to pxe")
 	}
 	// this stage will be called by baremetalInstance when pxe start notify
 	self.SetSSHStage(pxeBootTask.OnPXEBoot)

pkg/baremetal/tasks/basedeploy.go

@@ -17,6 +17,8 @@ package tasks
 import (
 	"context"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"
@@ -64,25 +66,25 @@ func (self *SBaremetalServerBaseDeployTask) OnPXEBoot(ctx context.Context, term
 	log.Infof("%s called on stage pxeboot, args: %v", self.GetName(), args)
 	result, err := self.serverDeployTask.DoDeploys(term)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Do deploy")
 	}
 	_, err = term.Run(
 		"/bin/sync",
 		"/sbin/sysctl -w vm.drop_caches=3",
 	)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Sync disk")
 	}
 	onFinishAction := self.GetFinishAction()
 	if utils.IsInStringArray(onFinishAction, []string{"restart", "shutdown"}) {
 		err = self.EnsurePowerShutdown(false)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "Ensure power off")
 		}
 		if onFinishAction == "restart" {
 			err = self.EnsurePowerUp()
 			if err != nil {
-				return err
+				return errors.Wrap(err, "Ensure power up")
 			}
 		}
 	}

pkg/baremetal/tasks/baseprepare.go

@@ -205,7 +205,7 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 				if lanConf.IPAddr == nic.IpAddr {
 					break
 				}
-				log.Infof("waiting IPMI DHCP address %s %s", lanConf.IPAddr, nic.IpAddr)
+				log.Infof("waiting IPMI DHCP address old:%s expect:%s", lanConf.IPAddr, nic.IpAddr)
 				tried += 2
 			}
 			if tried >= maxTries {
@@ -262,14 +262,15 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 	if err := task.sendStorageInfo(size); err != nil {
 		log.Errorf("sendStorageInfo error: %v", err)
 	}
-	for i := range nicsInfo {
-		if nicsInfo[i].Mac.String() == adminNic.GetMac().String() {
-			if i != 0 {
-				nicsInfo = append(nicsInfo[i:], nicsInfo[0:i]...)
-			}
-			break
-		}
-	}
+	// XXX do not change nic order anymore
+	// for i := range nicsInfo {
+	// 	if nicsInfo[i].Mac.String() == adminNic.GetMac().String() {
+	// 		if i != 0 {
+	// 			nicsInfo = append(nicsInfo[i:], nicsInfo[0:i]...)
+	// 		}
+	// 		break
+	// 	}
+	// }
 	err = task.removeAllNics()
 	if err != nil {
 		return err

pkg/baremetal/tasks/create.go

@@ -74,6 +74,7 @@ func (self *SBaremetalServerCreateTask) DoDeploys(term *ssh.Client) (jsonutils.J
 }
 
 func (self *SBaremetalServerCreateTask) onError(term *ssh.Client, err error) error {
+	log.Errorf("Create server error: %+v", err)
 	if err1 := self.Baremetal.GetServer().DoEraseDisk(term); err1 != nil {
 		log.Warningf("EraseDisk error: %v", err1)
 	}

pkg/baremetal/utils/detect_storages/detect_storages.go

@@ -36,8 +36,8 @@ func GetRaidDevices(drv raid.IRaidDriver) []*baremetal.BaremetalStorage {
 	return devs
 }
 
-func GetRaidLogicVolumes(drv raid.IRaidDriver) ([]int, error) {
-	lvs := []int{}
+func GetRaidLogicVolumes(drv raid.IRaidDriver) ([]*raid.RaidLogicalVolume, error) {
+	lvs := []*raid.RaidLogicalVolume{}
 	for _, adapter := range drv.GetAdapters() {
 		lv, err := adapter.GetLogicVolumes()
 		if err != nil {
@@ -50,12 +50,12 @@ func GetRaidLogicVolumes(drv raid.IRaidDriver) ([]int, error) {
 
 func DetectStorageInfo(term *ssh.Client, wait bool) ([]*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, error) {
 	raidDiskInfo := make([]*baremetal.BaremetalStorage, 0)
-	lvDiskInfo := make([]int, 0)
+	lvDiskInfo := make([]*raid.RaidLogicalVolume, 0)
 
 	raidDrivers := []string{}
 	for _, drv := range drivers.GetDrivers(term) {
 		if err := drv.ParsePhyDevs(); err != nil {
-			log.V(2).Warningf("ParsePhyDevs: %v", err)
+			log.Warningf("Raid driver %s ParsePhyDevs: %v", drv.GetName(), err)
 			continue
 		}
 		raidDiskInfo = append(raidDiskInfo, GetRaidDevices(drv)...)
@@ -70,8 +70,7 @@ func DetectStorageInfo(term *ssh.Client, wait bool) ([]*baremetal.BaremetalStora
 		raidDrivers = append(raidDrivers, drv.GetName())
 	}
 
-	log.Infof("Get Raid drivers: %v", raidDrivers)
-
+	log.Infof("Get Raid drivers: %v, collecting disks info ...", raidDrivers)
 	pcieRet, err := term.Run("/lib/mos/lsdisk --pcie")
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("Fail to retrieve PCIE DISK info")

pkg/baremetal/utils/disktool/disktool.go

@@ -19,8 +19,11 @@ import (
 	"math"
 	"strings"
 
+	"github.com/pkg/errors"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
+	raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	fileutils "yunion.io/x/onecloud/pkg/util/fileutils2"
@@ -205,6 +208,7 @@ func (p *Partition) GetSizeMB() (int64, error) {
 type DiskPartitions struct {
 	driver     string
 	adapter    int
+	raidConfig string
 	sizeMB     int64 // MB
 	tool       *PartitionTool
 	dev        string
@@ -217,10 +221,11 @@ type DiskPartitions struct {
 	partitions []*Partition
 }
 
-func newDiskPartitions(driver string, adapter int, sizeMB int64, blockSize int64, tool *PartitionTool) *DiskPartitions {
+func newDiskPartitions(driver string, adapter int, raidConfig string, sizeMB int64, blockSize int64, tool *PartitionTool) *DiskPartitions {
 	ps := new(DiskPartitions)
 	ps.driver = driver
 	ps.adapter = adapter
+	ps.raidConfig = raidConfig
 	ps.sizeMB = sizeMB
 	ps.tool = tool
 	ps.blockSize = blockSize
@@ -228,6 +233,15 @@ func newDiskPartitions(driver string, adapter int, sizeMB int64, blockSize int64
 	return ps
 }
 
+func (p *DiskPartitions) IsRaidDriver() bool {
+	return utils.IsInStringArray(p.driver, []string{
+		baremetal.DISK_DRIVER_MEGARAID,
+		baremetal.DISK_DRIVER_HPSARAID,
+		baremetal.DISK_DRIVER_MARVELRAID,
+		baremetal.DISK_DRIVER_MPT2SAS,
+	})
+}
+
 func (p *DiskPartitions) SetInfo(info *types.SDiskInfo) *DiskPartitions {
 	p.dev = fmt.Sprintf("/dev/%s", info.Dev)
 	p.devName = info.Dev
@@ -240,6 +254,20 @@ func (p *DiskPartitions) SetInfo(info *types.SDiskInfo) *DiskPartitions {
 	return p
 }
 
+func (p *DiskPartitions) ReInitInfo() error {
+	cmd := "/lib/mos/lsdisk"
+	lines, err := p.tool.Run(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "Disk %#v reset info list disk", p)
+	}
+	for _, disk := range sysutils.ParseDiskInfo(lines, p.driver) {
+		if disk.Dev == p.GetDevName() {
+			p.SetInfo(disk)
+		}
+	}
+	return p.RetrievePartitionInfo()
+}
+
 func (ps *DiskPartitions) MBSectors() int64 {
 	return int64(1024 * 1024 / ps.blockSize)
 }
@@ -264,7 +292,27 @@ func (ps *DiskPartitions) IsReady() bool {
 }
 
 func (ps *DiskPartitions) GetDevName() string {
-	return ps.devName
+	devName := ps.devName
+	if !ps.IsRaidDriver() || ps.raidConfig == baremetal.DISK_CONF_NONE {
+		return devName
+	}
+	raidDrv, err := raiddrivers.GetDriverWithInit(ps.driver, ps.tool.runner.Term())
+	if err != nil {
+		log.Errorf("Failed to find %s raid driver: %v", ps.driver, err)
+		return devName
+	}
+	// find first raid adapter logical volume
+	lv, err := raiddrivers.GetFirstLogicalVolume(raidDrv, ps.adapter)
+	if err != nil {
+		log.Errorf("Failed to find raid %s adapter %d first logical volume: %v", raidDrv.GetName(), ps.adapter, err)
+		return devName
+	}
+	if len(lv.BlockDev) == 0 {
+		log.Warningf("Raid %s adapter %d first logical volume block device is empty", raidDrv.GetName(), ps.adapter)
+		return devName
+	}
+	devName = strings.TrimLeft(lv.BlockDev, "/dev/")
+	return devName
 }
 
 func (ps *DiskPartitions) RetrievePartitionInfo() error {
@@ -312,6 +360,10 @@ func (ps *DiskPartitions) doResize(dev string, cmd string) error {
 	return ps.RetrievePartitionInfo()
 }
 
+func (ps *DiskPartitions) GetPartitions() []*Partition {
+	return ps.partitions
+}
+
 func (ps *DiskPartitions) ResizePartition(offsetMB int64) error {
 	if len(ps.partitions) == 0 {
 		return fmt.Errorf("ResizePartitions error: total %d partitions", len(ps.partitions))
@@ -512,6 +564,7 @@ func (ps *DiskPartitions) CreatePartition(sizeMB int64, fs string, doformat bool
 
 type IPartitionRunner interface {
 	Run(cmds ...string) ([]string, error)
+	Term() *ssh.Client
 }
 
 type PartitionTool struct {
@@ -543,6 +596,10 @@ func (tool *PartitionTool) DebugString() string {
 	return strings.Join(ret, "\n")
 }
 
+func (tool *PartitionTool) Disks() []*DiskPartitions {
+	return tool.disks
+}
+
 func (tool *PartitionTool) parseLsDisk(lines []string, driver string) {
 	disks := sysutils.ParseDiskInfo(lines, driver)
 	if len(disks) == 0 {
@@ -556,7 +613,7 @@ func (tool *PartitionTool) parseLsDisk(lines []string, driver string) {
 
 func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguration) *PartitionTool {
 	for _, d := range diskConfs {
-		disk := newDiskPartitions(d.Driver, d.Adapter, d.Size, d.Block, tool)
+		disk := newDiskPartitions(d.Driver, d.Adapter, d.RaidConfig, d.Size, d.Block, tool)
 		tool.disks = append(tool.disks, disk)
 		var key string
 		if d.Driver == baremetal.DISK_DRIVER_LINUX {
@@ -575,9 +632,9 @@ func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguratio
 }
 
 func (tool *PartitionTool) IsAllDisksReady() bool {
-	for _, d := range tool.disks {
+	for idx, d := range tool.disks {
 		if !d.IsReady() {
-			log.Errorf("disk %#v not ready", d)
+			log.Errorf("disk.%d %#v not ready", idx, d)
 			return false
 		}
 	}
@@ -681,3 +738,7 @@ func NewSSHPartitionTool(term *ssh.Client) *SSHPartitionTool {
 func (tool *SSHPartitionTool) Run(cmds ...string) ([]string, error) {
 	return tool.term.Run(cmds...)
 }
+
+func (tool *SSHPartitionTool) Term() *ssh.Client {
+	return tool.term
+}

pkg/baremetal/utils/ipmitool/ipmitool.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/stringutils"
@@ -444,7 +446,7 @@ func DoReboot(exector IPMIExecutor) error {
 	var err error
 	status, err = GetChassisPowerStatus(exector)
 	if err != nil {
-		return err
+		log.Errorf("DoReboot get power status 1st: %v", err)
 	}
 
 	isValidStatus := func(s string) bool {
@@ -455,25 +457,24 @@ func DoReboot(exector IPMIExecutor) error {
 		time.Sleep(1 * time.Second)
 		status, err = GetChassisPowerStatus(exector)
 		if err != nil {
-			return err
+			log.Errorf("DoReboot %d tries to get power status: %v", tried, err)
 		}
 	}
 
 	if !isValidStatus(status) {
-		return fmt.Errorf("Unexpected status: %s", status)
+		return fmt.Errorf("Unexpected power status: %q", status)
 	}
 
 	// do shutdown
 	if status == types.POWER_STATUS_ON {
-		err = DoHardShutdown(exector)
-		if err != nil {
-			return err
+		if err := DoHardShutdown(exector); err != nil {
+			log.Errorf("DoHardShutdown: %v", err)
 		}
 		time.Sleep(1 * time.Second)
 		for tried := 0; tried < maxTries; tried++ {
 			status, err = GetChassisPowerStatus(exector)
 			if err != nil {
-				return err
+				log.Errorf("DoReboot %d tries to get power status: %v", tried, err)
 			}
 			if status == types.POWER_STATUS_OFF {
 				break
@@ -483,28 +484,24 @@ func DoReboot(exector IPMIExecutor) error {
 	}
 
 	// do power on
-	status, err = GetChassisPowerStatus(exector)
-	if err != nil {
-		return err
-	}
+	status, _ = GetChassisPowerStatus(exector)
 	for tried := 0; status != types.POWER_STATUS_ON && tried < maxTries; tried++ {
-		err = DoPowerOn(exector)
-		if err != nil {
-			return err
+		if err := DoPowerOn(exector); err != nil {
+			log.Errorf("DoReboot %d tries to power on: %v", tried, err)
 		}
 		time.Sleep(1 * time.Second)
 		status, err = GetChassisPowerStatus(exector)
 		if err != nil {
-			return err
+			log.Errorf("DoReboot %d tries to get power status: %v", tried, err)
 		}
 	}
 
 	status, err = GetChassisPowerStatus(exector)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Get power status after power on")
 	}
 	if status != types.POWER_STATUS_ON {
-		return fmt.Errorf("do reboot fail to poweron, current status: %s", status)
+		return errors.Errorf("do reboot fail to poweron, current status: %s", status)
 	}
 	return nil
 }

pkg/baremetal/utils/raid/drivers/drivers.go

@@ -17,6 +17,10 @@ package drivers
 import (
 	"fmt"
 
+	"github.com/pkg/errors"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/hpssactl"
@@ -35,6 +39,30 @@ func GetDriver(name string, term *ssh.Client) raid.IRaidDriver {
 	return factory(term)
 }
 
+func GetDriverWithInit(name string, term *ssh.Client) (raid.IRaidDriver, error) {
+	drv := GetDriver(name, term)
+	if drv == nil {
+		return nil, errors.Errorf("Not found raid driver %q", name)
+	}
+	return drv, drv.ParsePhyDevs()
+}
+
+func GetDriverByKernelModule(module string, term *ssh.Client) (raid.IRaidDriver, error) {
+	name := ""
+	switch module {
+	case raid.MODULE_MEGARAID:
+		name = baremetal.DISK_DRIVER_MEGARAID
+	case raid.MODULE_HPSA:
+		name = baremetal.DISK_DRIVER_HPSARAID
+	case raid.MODULE_MPT2SAS, raid.MODULE_MPT3SAS:
+		name = baremetal.DISK_DRIVER_MPT2SAS
+	}
+	if name == "" {
+		return nil, errors.Errorf("Not support module %q", module)
+	}
+	return GetDriverWithInit(name, term)
+}
+
 func GetDrivers(term *ssh.Client) []raid.IRaidDriver {
 	ret := []raid.IRaidDriver{}
 	for _, factory := range raid.RaidDrivers {
@@ -111,3 +139,41 @@ func buildRaid(adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) erro
 	}
 	return nil
 }
+
+func GetFirstLogicalVolume(drv raid.IRaidDriver, adapterIdx int) (*raid.RaidLogicalVolume, error) {
+	var adapter raid.IRaidAdapter = nil
+	for _, ada := range drv.GetAdapters() {
+		if ada.GetIndex() == adapterIdx {
+			adapter = ada
+		}
+	}
+	if adapter == nil {
+		return nil, errors.Errorf("Not found raid %s adapter %d", drv.GetName(), adapterIdx)
+	}
+	lvs, err := adapter.GetLogicVolumes()
+	if err != nil {
+		return nil, errors.Wrapf(err, "Get adapter logical volume")
+	}
+	if len(lvs) == 0 {
+		return nil, errors.Errorf("Adapter %d empty logical volume", adapterIdx)
+	}
+	return lvs[0], nil
+}
+
+func GetBlockDeviceLogicalVolume(drv raid.IRaidDriver, blockDev string) (*raid.RaidLogicalVolume, error) {
+	lvs := make([]*raid.RaidLogicalVolume, 0)
+	for _, ada := range drv.GetAdapters() {
+		aLvs, err := ada.GetLogicVolumes()
+		if err != nil {
+			return nil, errors.Errorf("Get adapter %d logical volumes: %v", ada.GetIndex(), err)
+		}
+		lvs = append(lvs, aLvs...)
+	}
+	log.Debugf("Get logic volumes: %s", jsonutils.Marshal(lvs).String())
+	for _, lv := range lvs {
+		if lv.BlockDev == blockDev {
+			return lv, nil
+		}
+	}
+	return nil, errors.Errorf("Not found logical volume by block device %s", blockDev)
+}

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -20,7 +20,10 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/pkg/utils"
 
@@ -39,14 +42,19 @@ type HPSARaidPhyDev struct {
 func newHPSARaidPhyDev(addr string, adapter int, rotate bool) *HPSARaidPhyDev {
 	b := raid.NewRaidBasePhyDev(baremetal.DISK_DRIVER_HPSARAID)
 	b.Adapter = adapter
+	if rotate {
+		b.Rotate = tristate.True
+	} else {
+		b.Rotate = tristate.False
+	}
 	return &HPSARaidPhyDev{
 		RaidBasePhyDev: b,
 		addr:           addr,
 	}
 }
 
-func (dev *HPSARaidPhyDev) ToBaremetalStorage() *baremetal.BaremetalStorage {
-	s := dev.RaidBasePhyDev.ToBaremetalStorage()
+func (dev *HPSARaidPhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalStorage {
+	s := dev.RaidBasePhyDev.ToBaremetalStorage(index)
 	s.Addr = dev.addr
 
 	return s
@@ -133,16 +141,20 @@ func (adapter *HPSARaidAdaptor) ParsePhyDevs() error {
 		adapter.parsePhyDevs(ret, isRotate)
 		return nil
 	}
-	cmd1 := GetCommand("controller", "slot=%d", fmt.Sprintf("%d", adapter.index), "ssdphysicaldrive", "all", "show", "detail")
-	cmd2 := GetCommand("controller", "slot=%d", fmt.Sprintf("%d", adapter.index), "physicaldrive", "all", "show", "detail")
-	var err error
-	if err = parseByCmd(cmd1, false); err != nil {
-		log.Errorf("parsePhyDevs by cmd %q: %v", cmd1, err)
+	cmd1 := GetCommand("controller", fmt.Sprintf("slot=%d", adapter.index), "ssdphysicaldrive", "all", "show", "detail")
+	cmd2 := GetCommand("controller", fmt.Sprintf("slot=%d", adapter.index), "physicaldrive", "all", "show", "detail")
+	var err1 error
+	var err2 error
+	if err1 = parseByCmd(cmd1, false); err1 != nil {
+		err1 = errors.Errorf("parsePhyDevs by cmd %q: %v", cmd1, err1)
 	}
-	if err = parseByCmd(cmd2, true); err != nil {
-		log.Errorf("parsePhyDevs by cmd %q: %v", cmd1, err)
+	if err2 = parseByCmd(cmd2, true); err2 != nil {
+		err2 = errors.Errorf("parsePhyDevs by cmd %q: %v", cmd1, err2)
 	}
-	return err
+	if err1 != nil && err2 != nil {
+		return errors.Errorf("ssd: %v, hdd: %v", err1, err2)
+	}
+	return nil
 }
 
 func (adapter *HPSARaidAdaptor) parsePhyDevs(lines []string, isRotate bool) {
@@ -153,7 +165,7 @@ func (adapter *HPSARaidAdaptor) parsePhyDevs(lines []string, isRotate bool) {
 			phydev = newHPSARaidPhyDev(m["addr"], adapter.index, isRotate)
 		} else if phydev != nil && phydev.parseLine(line) && phydev.isComplete() {
 			oldDev := adapter.getPhyDevByAddr(phydev.addr)
-			if oldDev != nil {
+			if oldDev == nil {
 				adapter.devs = append(adapter.devs, phydev)
 			}
 			phydev = nil
@@ -172,8 +184,8 @@ func (adapter *HPSARaidAdaptor) getPhyDevByAddr(addr string) *HPSARaidPhyDev {
 
 func (adapter *HPSARaidAdaptor) GetDevices() []*baremetal.BaremetalStorage {
 	ret := []*baremetal.BaremetalStorage{}
-	for _, dev := range adapter.devs {
-		ret = append(ret, dev.ToBaremetalStorage())
+	for idx, dev := range adapter.devs {
+		ret = append(ret, dev.ToBaremetalStorage(idx))
 	}
 	return ret
 }
@@ -263,7 +275,7 @@ func (adapter *HPSARaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, c
 }
 
 func (adapter *HPSARaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
-	return adapter.buildRaid("10", devs, conf)
+	return adapter.buildRaid("1+0", devs, conf)
 }
 
 func (adapter *HPSARaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage) error {
@@ -285,7 +297,7 @@ func (adapter *HPSARaidAdaptor) removeLogicVolume(idx int) error {
 	return err
 }
 
-func (adapter *HPSARaidAdaptor) GetLogicVolumes() ([]int, error) {
+func (adapter *HPSARaidAdaptor) GetLogicVolumes() ([]*raid.RaidLogicalVolume, error) {
 	cmd := GetCommand("controller", fmt.Sprintf("slot=%d", adapter.index), "logicaldrive", "all", "show")
 	ret, err := adapter.raid.term.Run(cmd)
 	if err != nil {
@@ -294,8 +306,8 @@ func (adapter *HPSARaidAdaptor) GetLogicVolumes() ([]int, error) {
 	return adapter.parseLogicalVolumes(ret)
 }
 
-func (adapter *HPSARaidAdaptor) parseLogicalVolumes(lines []string) ([]int, error) {
-	lvs := []int{}
+func (adapter *HPSARaidAdaptor) parseLogicalVolumes(lines []string) ([]*raid.RaidLogicalVolume, error) {
+	lvs := []*raid.RaidLogicalVolume{}
 	for _, line := range lines {
 		m := regutils2.SubGroupMatch(`logicaldrive\s+(?P<addr>\w+)`, line)
 		if len(m) > 0 {
@@ -304,7 +316,10 @@ func (adapter *HPSARaidAdaptor) parseLogicalVolumes(lines []string) ([]int, erro
 			if err != nil {
 				return nil, fmt.Errorf("%s not int: %v", idxStr, err)
 			}
-			lvs = append(lvs, idx)
+			lvs = append(lvs, &raid.RaidLogicalVolume{
+				Index:   idx,
+				Adapter: adapter.index,
+			})
 		}
 	}
 	return lvs, nil
@@ -315,9 +330,9 @@ func (adapter *HPSARaidAdaptor) RemoveLogicVolumes() error {
 	if err != nil {
 		return fmt.Errorf("Failed to get logic volumes: %v", err)
 	}
-	for i := len(lvs) - 1; i >= 0; i-- {
-		if err := adapter.removeLogicVolume(i); err != nil {
-			return fmt.Errorf("Remove %d logical volume: %v", i, err)
+	for _, i := range raid.ReverseLogicalArray(lvs) {
+		if err := adapter.removeLogicVolume(i.Index); err != nil {
+			return fmt.Errorf("Remove %#v logical volume: %v", i, err)
 		}
 	}
 	return nil
@@ -336,7 +351,7 @@ func NewHPSARaid(term *ssh.Client) raid.IRaidDriver {
 }
 
 func (r *HPSARaid) ParsePhyDevs() error {
-	if !utils.IsInStringArray("hpsa", raid.GetModules(r.term)) {
+	if !utils.IsInStringArray(raid.MODULE_HPSA, raid.GetModules(r.term)) {
 		return fmt.Errorf("Not found hpsa module")
 	}
 	cmd := GetCommand("controller", "all", "show")

pkg/baremetal/utils/raid/interface.go

@@ -31,7 +31,7 @@ type IRaidDriver interface {
 type IRaidAdapter interface {
 	GetIndex() int
 	PreBuildRaid(confs []*api.BaremetalDiskConfig) error
-	GetLogicVolumes() ([]int, error)
+	GetLogicVolumes() ([]*RaidLogicalVolume, error)
 	RemoveLogicVolumes() error
 	GetDevices() []*baremetal.BaremetalStorage
 

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/stringutils"
@@ -60,8 +61,8 @@ func NewMegaRaidPhyDev() *MegaRaidPhyDev {
 	}
 }
 
-func (dev *MegaRaidPhyDev) ToBaremetalStorage() *baremetal.BaremetalStorage {
-	s := dev.RaidBasePhyDev.ToBaremetalStorage()
+func (dev *MegaRaidPhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalStorage {
+	s := dev.RaidBasePhyDev.ToBaremetalStorage(index)
 	s.Enclosure = dev.enclosure
 	s.Slot = dev.slot
 	s.Size = dev.GetSize()
@@ -182,16 +183,130 @@ func GetSpecString(dev *baremetal.BaremetalStorage) string {
 }
 
 type MegaRaidAdaptor struct {
-	index int
-	raid  *MegaRaid
-	devs  []*MegaRaidPhyDev
+	index        int
+	storcliIndex int
+	raid         *MegaRaid
+	devs         []*MegaRaidPhyDev
+	sn           string
+	busNumber    string
+	deviceNumber string
+	funcNumber   string
+	// used by sg_map
+	hostNum int
+	//channelNum int
 }
 
-func NewMegaRaidAdaptor(index int, raid *MegaRaid) *MegaRaidAdaptor {
-	return &MegaRaidAdaptor{
-		index: index,
-		raid:  raid,
+func NewMegaRaidAdaptor(index int, raid *MegaRaid) (*MegaRaidAdaptor, error) {
+	adapter := &MegaRaidAdaptor{
+		index:        index,
+		storcliIndex: -1,
+		raid:         raid,
 	}
+	if err := adapter.fillInfo(); err != nil {
+		return adapter, errors.Wrapf(err, "%d fill info", adapter.index)
+	}
+	return adapter, nil
+}
+
+func (adapter *MegaRaidAdaptor) fillInfo() error {
+	cmd := GetCommand("-AdpAllInfo", fmt.Sprintf("-a%d", adapter.index))
+	ret, err := adapter.remoteRun(cmd)
+	if err != nil {
+		return errors.Wrap(err, "remote get SN")
+	}
+	for _, l := range ret {
+		key, val := stringutils.SplitKeyValue(l)
+		if len(key) == 0 {
+			continue
+		}
+		switch key {
+		case "Serial No":
+			adapter.sn = val
+		}
+	}
+	if len(adapter.sn) == 0 {
+		return errors.New("Not found Serial No")
+	}
+	return adapter.fillPCIInfo()
+}
+
+func (adapter *MegaRaidAdaptor) fillPCIInfo() error {
+	cmd := GetCommand("-adpgetpciinfo", fmt.Sprintf("-a%d", adapter.index))
+	ret, err := adapter.remoteRun(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "%d remote run get pci info", adapter.index)
+	}
+	for _, l := range ret {
+		key, val := stringutils.SplitKeyValue(l)
+		if len(key) == 0 {
+			continue
+		}
+		switch key {
+		case "Bus Number":
+			if len(val) == 1 {
+				val = fmt.Sprintf("0%s", val)
+			}
+			if len(val) != 2 {
+				return errors.Errorf("Invalid bus number: %s", val)
+			}
+			adapter.busNumber = val
+		case "Device Number":
+			if len(val) == 1 {
+				val = fmt.Sprintf("0%s", val)
+			}
+			if len(val) != 2 {
+				return errors.Errorf("Invalid device number: %s", val)
+			}
+			adapter.deviceNumber = val
+		case "Function Number":
+			if len(val) != 1 {
+				return errors.Errorf("Invalid function number: %s", val)
+			}
+			adapter.funcNumber = val
+		}
+	}
+	if len(adapter.busNumber) == 0 || len(adapter.deviceNumber) == 0 || len(adapter.funcNumber) == 0 {
+		return errors.New("Not found bus number")
+	}
+	pciDir := fmt.Sprintf("/sys/bus/pci/devices/0000:%s:%s.%s/", adapter.busNumber, adapter.deviceNumber, adapter.funcNumber)
+	cmd = raiddrivers.GetCommand("ls", pciDir, "|", "grep", "host")
+	ret, err = adapter.remoteRun(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "find pci host number")
+	}
+	if len(ret) == 0 {
+		return errors.Errorf("Not find pci host dir")
+	}
+	hostNumStr := ret[0]
+	hostNum, err := strconv.Atoi(strings.TrimLeft(hostNumStr, "host"))
+	if err != nil {
+		return errors.Errorf("Invalid hostNum %s", hostNumStr)
+	}
+	adapter.hostNum = hostNum
+	pciHostDir := fmt.Sprintf("%s%s/", pciDir, hostNumStr)
+	// $ ls /sys/bus/pci/devices/0000:03:00.0/host0/ | grep target | head -n 1
+	// target0:2:0
+	targetCmd := raiddrivers.GetCommand("ls", pciHostDir, "|", "grep", "target", "|", "head", "-n", "1")
+	ret, err = adapter.remoteRun(targetCmd)
+	if err != nil {
+		return errors.Wrapf(err, "find target %q", targetCmd)
+	}
+	if len(ret) == 0 {
+		return errors.Errorf("Not find target dir")
+	}
+	//targetStr := ret[0]
+	//parts := strings.Split(targetStr, ":")
+	//if len(parts) != 3 {
+	//// not build raid logical volume yet
+	//log.Warningf("Cmd %q invalid target string %q, skip fill logical volume info", targetCmd, targetStr)
+	//return nil
+	//}
+	//channelNum, err := strconv.Atoi(parts[1])
+	//if err != nil {
+	//return errors.Errorf("Invalid channel number %s", parts[1])
+	//}
+	//adapter.channelNum = channelNum
+	return nil
 }
 
 func (adapter *MegaRaidAdaptor) GetIndex() int {
@@ -213,31 +328,61 @@ func (adapter *MegaRaidAdaptor) AddPhyDev(dev *MegaRaidPhyDev) {
 
 func (adapter *MegaRaidAdaptor) GetDevices() []*baremetal.BaremetalStorage {
 	ret := []*baremetal.BaremetalStorage{}
-	for _, dev := range adapter.devs {
-		ret = append(ret, dev.ToBaremetalStorage())
+	for idx, dev := range adapter.devs {
+		ret = append(ret, dev.ToBaremetalStorage(idx))
 	}
 	return ret
 }
 
-func (adapter *MegaRaidAdaptor) GetLogicVolumes() ([]int, error) {
+func (adapter *MegaRaidAdaptor) GetLogicVolumes() ([]*raiddrivers.RaidLogicalVolume, error) {
 	cmd := GetCommand("-LDInfo", "-Lall", fmt.Sprintf("-a%d", adapter.index))
 	ret, err := adapter.remoteRun(cmd)
 	if err != nil {
 		return nil, fmt.Errorf("GetLogicVolumes error: %v", err)
 	}
-	return adapter.parseLogicVolumes(ret), nil
+	return adapter.parseLogicVolumes(ret)
 }
 
-func (adapter *MegaRaidAdaptor) parseLogicVolumes(lines []string) []int {
-	lvIdx := []int{}
+var logicalVolumeIdRegexp = regexp.MustCompile(`.*(Target Id: (?P<idx>[0-9]+))`)
+
+func (adapter *MegaRaidAdaptor) parseLogicVolumes(lines []string) ([]*raiddrivers.RaidLogicalVolume, error) {
+	lvs := make([]*raiddrivers.RaidLogicalVolume, 0)
 	for _, line := range lines {
 		key, val := stringutils.SplitKeyValue(line)
 		if key != "" && key == "Virtual Drive" {
-			idx, _ := strconv.Atoi(strings.Split(val, " ")[0])
-			lvIdx = append(lvIdx, idx)
+			idxStr := regutils2.GetParams(logicalVolumeIdRegexp, val)["idx"]
+			idx, err := strconv.Atoi(idxStr)
+			if err != nil {
+				return nil, errors.Errorf("index %q to int: %v", idxStr, err)
+			}
+			blockDev, err := getLogicVolumeDeviceById(adapter.hostNum, idx, adapter.getTerm())
+			if err != nil {
+				return nil, err
+			}
+			lvs = append(lvs, &raiddrivers.RaidLogicalVolume{
+				Index:    idx,
+				Adapter:  adapter.index,
+				BlockDev: blockDev,
+			})
 		}
 	}
-	return lvIdx
+	return lvs, nil
+}
+
+func getLogicVolumeDeviceById(hostNum, scsiId int, term *ssh.Client) (string, error) {
+	items, err := raiddrivers.SGMap(term)
+	if err != nil {
+		return "", err
+	}
+	isMatch := func(item api.SGMapItem) bool {
+		return item.HostNumber == hostNum && item.SCSIId == scsiId
+	}
+	for _, item := range items {
+		if isMatch(item) {
+			return item.LinuxDeviceName, nil
+		}
+	}
+	return "", errors.Errorf("Not found SG item by id: %d:%d", hostNum, scsiId)
 }
 
 func (adapter *MegaRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error {
@@ -384,7 +529,6 @@ func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalSt
 
 func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig, level uint) error {
 	args := []string{}
-	args = append(args, fmt.Sprintf("/c%d", adapter.index))
 	args = append(args, "add", "vd", fmt.Sprintf("type=r%d", level))
 	args = append(args, adapter.conf2ParamsStorcliSize(conf)...)
 	labels := []string{}
@@ -396,9 +540,12 @@ func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStor
 		args = append(args, "PDperArray=2")
 	}
 	args = append(args, adapter.conf2ParamsStorcli(conf)...)
-	cmd := GetCommand2(args...)
+	cmd, err := adapter.GetStorcliCommand(args...)
+	if err != nil {
+		return errors.Wrapf(err, "build raid %d", level)
+	}
 	log.Infof("_storcliBuildRaid command: %s", cmd)
-	_, err := adapter.remoteRun(cmd)
+	_, err = adapter.remoteRun(cmd)
 	return err
 }
 
@@ -451,8 +598,86 @@ func (adapter *MegaRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage
 	return cliBuildRaid(devs, nil, adapter.megacliBuildNoRaid, adapter.storcliBuildNoRaid)
 }
 
+type StorcliAdaptor struct {
+	Controller int
+	SN         string
+}
+
+func newStorcliAdaptor() *StorcliAdaptor {
+	return &StorcliAdaptor{
+		Controller: -1,
+		SN:         "",
+	}
+}
+
+func (a *StorcliAdaptor) isComplete() bool {
+	return a.Controller >= 0 && a.SN != ""
+}
+
+func (a *StorcliAdaptor) parseLine(l string) {
+	parts := strings.Split(l, " = ")
+	if len(parts) != 2 {
+		return
+	}
+	key := parts[0]
+	val := parts[1]
+	switch key {
+	case "Controller":
+		a.Controller, _ = strconv.Atoi(val)
+	case "Serial Number":
+		a.SN = val
+	}
+}
+
+func (raid *MegaRaid) GetStorcliAdaptor() (map[string]*StorcliAdaptor, error) {
+	ret := make(map[string]*StorcliAdaptor)
+	cmd := GetCommand2("/call", "show", "|", "grep", "-iE", `'^(Controller|Serial Number)\s='`)
+	lines, err := raid.term.Run(cmd)
+	if err != nil {
+		return nil, errors.Wrap(err, "Get storcli adapter")
+	}
+	adapter := newStorcliAdaptor()
+	for _, l := range lines {
+		adapter.parseLine(l)
+		if adapter.isComplete() {
+			ret[adapter.SN] = adapter
+			adapter = newStorcliAdaptor()
+		}
+	}
+	return ret, nil
+}
+
+func (adapter *MegaRaidAdaptor) storcliCtrlIndex() (int, error) {
+	if adapter.storcliIndex >= 0 {
+		return adapter.storcliIndex, nil
+	}
+	storcliAdaps, err := adapter.raid.GetStorcliAdaptor()
+	if err != nil {
+		return -1, errors.Wrap(err, "Get all Storcli adaptor")
+	}
+	storAdap, ok := storcliAdaps[adapter.sn]
+	if !ok {
+		return -1, errors.Errorf("Not found storcli adaptor by SN %q", adapter.sn)
+	}
+	return storAdap.Controller, nil
+}
+
+func (adapter *MegaRaidAdaptor) GetStorcliCommand(args ...string) (string, error) {
+	controllerIdx, err := adapter.storcliCtrlIndex()
+	if err != nil {
+		return "", errors.Errorf("Adapter %d get storcli controller index: %v", adapter.index, err)
+	}
+	nargs := []string{fmt.Sprintf("/c%d", controllerIdx)}
+	nargs = append(nargs, args...)
+	return GetCommand2(nargs...), nil
+}
+
 func (adapter *MegaRaidAdaptor) storcliIsJBODEnabled() bool {
-	cmd := GetCommand2(fmt.Sprintf("/c%d", adapter.index), "show", "jbod")
+	cmd, err := adapter.GetStorcliCommand("show", "jbod")
+	if err != nil {
+		log.Errorf("get storcli controller cmd: %v", err)
+		return false
+	}
 	lines, err := adapter.remoteRun(cmd)
 	if err != nil {
 		log.Errorf("storcliIsJBODEnabled error: %s", err)
@@ -476,8 +701,12 @@ func (adapter *MegaRaidAdaptor) storcliEnableJBOD(enable bool) bool {
 	if enable {
 		val = "on"
 	}
-	cmd := GetCommand2(fmt.Sprintf("/c%d", adapter.index), "set", fmt.Sprintf("jbod=%s", val))
-	_, err := adapter.remoteRun(cmd)
+	cmd, err := adapter.GetStorcliCommand("set", fmt.Sprintf("jbod=%s", val))
+	if err != nil {
+		log.Errorf("get storcli controller cmd: %v", err)
+		return false
+	}
+	_, err = adapter.remoteRun(cmd)
 	if err != nil {
 		log.Errorf("EnableJBOD %v fail: %v", enable, err)
 		return false
@@ -496,7 +725,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildJBOD(devs []*baremetal.BaremetalStor
 	}
 	cmds := []string{}
 	for _, d := range devs {
-		cmd := GetCommand2(fmt.Sprintf("/c%d/e%d/s%d", adapter.index, d.Enclosure, d.Slot))
+		cmd := GetCommand2(fmt.Sprintf("/c%d/e%d/s%d", adapter.storcliIndex, d.Enclosure, d.Slot))
 		cmds = append(cmds, cmd)
 	}
 	log.Infof("storcliBuildJBOD cmds: %v", cmds)
@@ -518,12 +747,14 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt
 		labels = append(labels, GetSpecString(dev))
 	}
 	args := []string{
-		fmt.Sprintf("/c%d", adapter.index),
 		"add", "vd", "each", "type=raid0",
 		fmt.Sprintf("drives=%s", strings.Join(labels, ",")),
 		"wt", "nora", "direct",
 	}
-	cmd := GetCommand2(args...)
+	cmd, err := adapter.GetStorcliCommand(args...)
+	if err != nil {
+		return errors.Wrapf(err, "build none raid")
+	}
 	_, err = adapter.remoteRun(cmd)
 	return err
 }
@@ -602,8 +833,8 @@ func (adapter *MegaRaidAdaptor) RemoveLogicVolumes() error {
 	if err != nil {
 		return err
 	}
-	for i := len(lvIdx) - 1; i >= 0; i-- {
-		cmd := GetCommand("-CfgLdDel", fmt.Sprintf("-L%d", i), "-Force", fmt.Sprintf("-a%d", adapter.index))
+	for _, i := range raiddrivers.ReverseLogicalArray(lvIdx) {
+		cmd := GetCommand("-CfgLdDel", fmt.Sprintf("-L%d", i.Index), "-Force", fmt.Sprintf("-a%d", adapter.index))
 		cmds = append(cmds, cmd)
 	}
 	if len(cmds) > 0 {
@@ -630,8 +861,8 @@ def _storcli_clear_jbod_disks(self):
 
 func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
 	devIds := []string{}
-	for _, dev := range adapter.devs {
-		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage()))
+	for idx, dev := range adapter.devs {
+		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage(idx)))
 	}
 	cmd := GetCommand("-PDMakeGood", fmt.Sprintf("-PhysDrv[%s]", strings.Join(devIds, ",")), "-Force", fmt.Sprintf("-a%d", adapter.index))
 	_, err := adapter.remoteRun(cmd)
@@ -674,7 +905,7 @@ func (raid *MegaRaid) GetName() string {
 }
 
 func (raid *MegaRaid) ParsePhyDevs() error {
-	if !utils.IsInStringArray("megaraid_sas", raiddrivers.GetModules(raid.term)) {
+	if !utils.IsInStringArray(raiddrivers.MODULE_MEGARAID, raiddrivers.GetModules(raid.term)) {
 		return fmt.Errorf("Not found megaraid_sas module")
 	}
 	cmd := GetCommand("-PDList", "-aALL")
@@ -692,18 +923,15 @@ func (raid *MegaRaid) ParsePhyDevs() error {
 func (raid *MegaRaid) parsePhyDevs(lines []string) error {
 	phyDev := NewMegaRaidPhyDev()
 	var adapter *MegaRaidAdaptor
+	var err error
 	for _, line := range lines {
-		matches := adapterPatter.FindStringSubmatch(line)
-		if len(matches) != 0 {
-			paramsMap := make(map[string]string)
-			for i, name := range sizePattern.SubexpNames() {
-				if i > 0 && i <= len(matches) {
-					paramsMap[name] = matches[i]
-				}
-			}
-			adapterStr := paramsMap["idx"]
+		adapterStr := regutils2.GetParams(adapterPatter, line)["idx"]
+		if adapterStr != "" {
 			adapterInt, _ := strconv.Atoi(adapterStr)
-			adapter = NewMegaRaidAdaptor(adapterInt, raid)
+			adapter, err = NewMegaRaidAdaptor(adapterInt, raid)
+			if err != nil {
+				return errors.Wrapf(err, "New raid adapter %d", adapterInt)
+			}
 			raid.adapters = append(raid.adapters, adapter)
 		} else if phyDev.parseLine(line) && phyDev.isComplete() {
 			if adapter == nil {
@@ -715,43 +943,30 @@ func (raid *MegaRaid) parsePhyDevs(lines []string) error {
 			phyDev = NewMegaRaidPhyDev()
 		}
 	}
-	for _, dev := range adapter.devs {
-		if err := raid.addPhyDevStripSize(dev); err != nil {
-			return fmt.Errorf("addPhyDevStripSize: %v", err)
+	for _, adapter := range raid.adapters {
+		if err := adapter.addPhyDevsStripSize(); err != nil {
+			log.Errorf("Adapter %d fill phsical devices strip size: %v", adapter.GetIndex(), err)
 		}
 	}
 	return nil
 }
 
-func (raid *MegaRaid) addPhyDevStripSize(phyDev *MegaRaidPhyDev) error {
+func (adapter *MegaRaidAdaptor) addPhyDevsStripSize() error {
 	grepCmd := []string{"grep", "-iE", "'^(Min|Max) Strip Size'"}
-	args := []string{"-adpallinfo", "-aall", "|"}
+	args := []string{"-adpallinfo", fmt.Sprintf("-a%d", adapter.index), "|"}
 	args = append(args, grepCmd...)
 	cmd := GetCommand(args...)
-	ret, err := raid.term.Run(cmd)
+	ret, err := adapter.remoteRun(cmd)
 	if err != nil {
 		return fmt.Errorf("addPhyDevStripSize error: %v", err)
 	}
-	return phyDev.parseStripSize(ret)
-}
-
-/*func (raid *MegaRaid) GetPhyDevs() []*MegaRaidPhyDev {
-	devs := make([]*MegaRaidPhyDev, 0)
-	for _, ada := range raid.adapters {
-		devs = append(devs, ada.GetPhyDevs()...)
+	for _, dev := range adapter.devs {
+		if err := dev.parseStripSize(ret); err != nil {
+			return err
+		}
 	}
-	return devs
-}*/
-
-//func (raid *MegaRaid) ParseLogicVolumes() bool {
-//cmd := GetCommand("-LDInfo", "-Lall", "-aALL")
-//ret, err := raid.term.Run(cmd)
-//if err != nil {
-//return false
-//}
-//raid.parseLogicVolumes(ret)
-//return true
-//}
+	return nil
+}
 
 func (raid *MegaRaid) CleanRaid() error {
 	for _, adapter := range raid.adapters {

pkg/baremetal/utils/raid/mvcli/mvcli.go

@@ -87,8 +87,8 @@ func (dev *MarvelRaidPhyDev) isComplete() bool {
 	return true
 }
 
-func (dev *MarvelRaidPhyDev) ToBaremetalStorage() *baremetal.BaremetalStorage {
-	s := dev.RaidBasePhyDev.ToBaremetalStorage()
+func (dev *MarvelRaidPhyDev) ToBaremetalStorage(idx int) *baremetal.BaremetalStorage {
+	s := dev.RaidBasePhyDev.ToBaremetalStorage(idx)
 	s.Slot = dev.slot
 	return s
 }
@@ -137,13 +137,13 @@ func (adapter *MarvelRaidAdaptor) parsePhyDevs(lines []string) error {
 
 func (adapter *MarvelRaidAdaptor) GetDevices() []*baremetal.BaremetalStorage {
 	ret := []*baremetal.BaremetalStorage{}
-	for _, dev := range adapter.devs {
-		ret = append(ret, dev.ToBaremetalStorage())
+	for idx, dev := range adapter.devs {
+		ret = append(ret, dev.ToBaremetalStorage(idx))
 	}
 	return ret
 }
 
-func (adapter *MarvelRaidAdaptor) GetLogicVolumes() ([]int, error) {
+func (adapter *MarvelRaidAdaptor) GetLogicVolumes() ([]*raid.RaidLogicalVolume, error) {
 	cmd := GetCommand("info", "-o", "vd")
 	ret, err := adapter.raid.term.Run(cmd)
 	if err != nil {
@@ -152,9 +152,9 @@ func (adapter *MarvelRaidAdaptor) GetLogicVolumes() ([]int, error) {
 	return adapter.parseLogicVolumes(ret)
 }
 
-func (adapter *MarvelRaidAdaptor) parseLogicVolumes(lines []string) ([]int, error) {
-	lvIdx := []int{}
-	usedDevs := []int{}
+func (adapter *MarvelRaidAdaptor) parseLogicVolumes(lines []string) ([]*raid.RaidLogicalVolume, error) {
+	lvIdx := make([]*raid.RaidLogicalVolume, 0)
+	usedDevs := make([]*raid.RaidLogicalVolume, 0)
 	for _, line := range lines {
 		key, val := stringutils.SplitKeyValue(line)
 		if key != "" {
@@ -163,14 +163,20 @@ func (adapter *MarvelRaidAdaptor) parseLogicVolumes(lines []string) ([]int, erro
 				if err != nil {
 					return nil, err
 				}
-				lvIdx = append(lvIdx, idx)
+				lvIdx = append(lvIdx, &raid.RaidLogicalVolume{
+					Index:   idx,
+					Adapter: adapter.index,
+				})
 			} else if key == "PD RAID setup" {
 				for _, d := range strings.Split(val, " ") {
 					idx, err := strconv.Atoi(d)
 					if err != nil {
 						return nil, err
 					}
-					usedDevs = append(usedDevs, idx)
+					usedDevs = append(usedDevs, &raid.RaidLogicalVolume{
+						Index:   idx,
+						Adapter: adapter.index,
+					})
 				}
 			}
 		}
@@ -186,9 +192,9 @@ func (adapter *MarvelRaidAdaptor) RemoveLogicVolumes() error {
 	if err != nil {
 		return fmt.Errorf("Failed to get logic volumes: %v", err)
 	}
-	for i := len(lvs) - 1; i >= 0; i-- {
-		if err := adapter.removeLogicVolume(i); err != nil {
-			return fmt.Errorf("Remove %d logical volume: %v", i, err)
+	for _, i := range raid.ReverseLogicalArray(lvs) {
+		if err := adapter.removeLogicVolume(i.Index); err != nil {
+			return fmt.Errorf("Remove %#v logical volume: %v", i, err)
 		}
 	}
 	return nil

pkg/baremetal/utils/raid/raid.go

@@ -17,11 +17,27 @@ package raid
 import (
 	"strings"
 
+	"github.com/pkg/errors"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
 
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/ssh"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
+)
+
+const (
+	MODULE_MEGARAID = "megaraid_sas"
+	MODULE_HPSA     = "hpsa"
+	MODULE_MPT2SAS  = "mpt2sas"
+	MODULE_MPT3SAS  = "mpt3sas"
+)
+
+const (
+	MaxUint                   = ^uint(0)
+	MaxInt                    = int(MaxUint >> 1)
+	UnknownLogicalVolumeIndex = MaxInt
 )
 
 type RaidDriverFactory func(term *ssh.Client) IRaidDriver
@@ -74,7 +90,7 @@ func (dev *RaidBasePhyDev) IsComplete() bool {
 	return true
 }
 
-func (dev *RaidBasePhyDev) ToBaremetalStorage() *baremetal.BaremetalStorage {
+func (dev *RaidBasePhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalStorage {
 	return &baremetal.BaremetalStorage{
 		Adapter: dev.Adapter,
 		Status:  dev.Status,
@@ -103,3 +119,28 @@ func GetModules(term *ssh.Client) []string {
 	}
 	return ret
 }
+
+func ReverseLogicalArray(input []*RaidLogicalVolume) []*RaidLogicalVolume {
+	s := make([]*RaidLogicalVolume, len(input))
+	for i := range input {
+		s[i] = input[i]
+	}
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+	return s
+}
+
+type RaidLogicalVolume struct {
+	Index    int
+	Adapter  int
+	BlockDev string
+}
+
+func SGMap(term *ssh.Client) ([]compute.SGMapItem, error) {
+	lines, err := term.Run("/usr/bin/sg_map -x")
+	if err != nil {
+		return nil, errors.Wrap(err, "run sg_map")
+	}
+	return sysutils.ParseSGMap(lines), nil
+}

pkg/baremetal/utils/raid/raid_test.go

@@ -0,0 +1,52 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package raid
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestReverseLogicalArray(t *testing.T) {
+	tests := []struct {
+		name  string
+		input []*RaidLogicalVolume
+		want  []*RaidLogicalVolume
+	}{
+		{
+			name:  "empty input",
+			input: []*RaidLogicalVolume{},
+			want:  []*RaidLogicalVolume{},
+		},
+		{
+			name: "reverse",
+			input: []*RaidLogicalVolume{
+				{Index: 1},
+				{Index: 2},
+			},
+			want: []*RaidLogicalVolume{
+				{Index: 2},
+				{Index: 1},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ReverseLogicalArray(tt.input); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ReverseIntArray() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -115,8 +115,8 @@ func (dev *Mpt2SASRaidPhyDev) isComplete() bool {
 	return true
 }
 
-func (dev *Mpt2SASRaidPhyDev) ToBaremetalStorage() *baremetal.BaremetalStorage {
-	s := dev.RaidBasePhyDev.ToBaremetalStorage()
+func (dev *Mpt2SASRaidPhyDev) ToBaremetalStorage(idx int) *baremetal.BaremetalStorage {
+	s := dev.RaidBasePhyDev.ToBaremetalStorage(idx)
 	s.Slot = dev.slot
 	s.Enclosure = dev.enclosure
 	s.Block = int64(dev.block)
@@ -176,13 +176,13 @@ func (adapter *Mpt2SASRaidAdaptor) parsePhyDevs(lines []string) error {
 
 func (adapter *Mpt2SASRaidAdaptor) GetDevices() []*baremetal.BaremetalStorage {
 	ret := []*baremetal.BaremetalStorage{}
-	for _, dev := range adapter.devs {
-		ret = append(ret, dev.ToBaremetalStorage())
+	for idx, dev := range adapter.devs {
+		ret = append(ret, dev.ToBaremetalStorage(idx))
 	}
 	return ret
 }
 
-func (adapter *Mpt2SASRaidAdaptor) GetLogicVolumes() ([]int, error) {
+func (adapter *Mpt2SASRaidAdaptor) GetLogicVolumes() ([]*raid.RaidLogicalVolume, error) {
 	cmd := adapter.raid.GetCommand(fmt.Sprintf("%d", adapter.index), "DISPLAY")
 	ret, err := adapter.raid.term.Run(cmd)
 	if err != nil {
@@ -191,24 +191,33 @@ func (adapter *Mpt2SASRaidAdaptor) GetLogicVolumes() ([]int, error) {
 	return adapter.parseLogicVolumes(ret)
 }
 
-func (adapter *Mpt2SASRaidAdaptor) parseLogicVolumes(lines []string) ([]int, error) {
-	lvIdx := []int{}
-	usedDevs := []int{}
+func (adapter *Mpt2SASRaidAdaptor) parseLogicVolumes(lines []string) ([]*raid.RaidLogicalVolume, error) {
+	lvIdx := []*raid.RaidLogicalVolume{}
+	usedDevs := []*raid.RaidLogicalVolume{}
 	for _, line := range lines {
 		key, val := stringutils.SplitKeyValue(line)
 		if key != "" && key == "Volume ID" {
 			idx, _ := strconv.Atoi(val)
-			lvIdx = append(lvIdx, idx)
+			lvIdx = append(lvIdx, &raid.RaidLogicalVolume{
+				Index:   idx,
+				Adapter: adapter.index,
+			})
 		} else if regexp.MustCompile(`PHY\[\d+\] Enclosure#/Slot#`).MatchString(key) {
 			idx, _ := strconv.Atoi(val)
-			usedDevs = append(usedDevs, idx)
+			usedDevs = append(usedDevs, &raid.RaidLogicalVolume{
+				Index:   idx,
+				Adapter: adapter.index,
+			})
 		}
 	}
 	if len(adapter.devs) < len(usedDevs) {
 		return nil, fmt.Errorf("adapter current dev %d < usedDevs %d", len(adapter.devs), len(usedDevs))
 	}
 	for i := 0; i < len(adapter.devs)-len(usedDevs); i++ {
-		lvIdx = append(lvIdx, -1)
+		lvIdx = append(lvIdx, &raid.RaidLogicalVolume{
+			Index:   -1,
+			Adapter: adapter.index,
+		})
 	}
 	return lvIdx, nil
 }
@@ -231,8 +240,8 @@ func (adapter *Mpt2SASRaidAdaptor) setBootIR() error {
 	if err != nil {
 		return err
 	}
-	if len(lvs) > 0 && lvs[0] > 0 {
-		args := []string{fmt.Sprintf("%d", adapter.index), "BOOTIR", fmt.Sprintf("%d", lvs[0])}
+	if len(lvs) > 0 && lvs[0].Index > 0 {
+		args := []string{fmt.Sprintf("%d", adapter.index), "BOOTIR", fmt.Sprintf("%d", lvs[0].Index)}
 		cmd := adapter.raid.GetCommand(args...)
 		_, err := adapter.raid.term.Run(cmd)
 		return err
@@ -315,9 +324,9 @@ func (r *Mpt2SASRaid) GetName() string {
 }
 
 func (r *Mpt2SASRaid) ParsePhyDevs() error {
-	if r.modulePCIProbed("mpt2sas") {
+	if r.modulePCIProbed(raid.MODULE_MPT2SAS) {
 		r.utility = "/opt/lsi/sas2ircu"
-	} else if r.modulePCIProbed("mpt3sas") {
+	} else if r.modulePCIProbed(raid.MODULE_MPT3SAS) {
 		r.utility = "/opt/lsi/sas3ircu"
 	} else {
 		return fmt.Errorf("Not probe mpt2sas or mpt3sas kernel module")

pkg/cloudcommon/cmdline/parser.go

@@ -109,6 +109,8 @@ func ParseDiskConfig(diskStr string, idx int) (*compute.DiskConfig, error) {
 			diskConfig.Mountpoint = p
 		} else if p == "autoextend" {
 			diskConfig.SizeMb = -1
+		} else if utils.IsInStringArray(p, compute.STORAGE_ALL_TYPES) {
+			diskConfig.Backend = p
 		} else if strings.HasPrefix(p, "snapshot-") {
 			// HACK: use snapshot creat disk format snapshot-id
 			// example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b

pkg/cloudcommon/db/count.go

@@ -0,0 +1,32 @@
+package db
+
+import (
+	"reflect"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/utils"
+)
+
+func FetchModelExtraCountProperties(model IModel, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	selfValue := reflect.ValueOf(model)
+	selfType := reflect.TypeOf(model)
+	for i := 0; i < selfValue.NumMethod(); i += 1 {
+		methodValue := selfValue.Method(i)
+		methodType := methodValue.Type()
+		if methodType.NumIn() != 0 || methodType.NumOut() != 2 {
+			continue
+		}
+		methodName := selfType.Method(i).Name
+		tokens := utils.CamelSplitTokens(methodName)
+		if len(tokens) < 3 {
+			continue
+		}
+		if strings.EqualFold(tokens[0], "get") && strings.EqualFold(tokens[len(tokens)-1], "count") {
+			resName := strings.ToLower(strings.Join(tokens[1:], "_"))
+			outs := methodValue.Call([]reflect.Value{})
+			extra.Add(jsonutils.NewInt(outs[0].Int()), resName)
+		}
+	}
+	return extra
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -466,15 +466,25 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if err != nil {
 		return nil, err
 	}
-	totalCnt := int64(q.Count())
+	totalCnt, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	// log.Debugf("total count %d", totalCnt)
 	if totalCnt == 0 {
 		emptyList := modules.ListResult{Data: []jsonutils.JSONObject{}}
 		return &emptyList, nil
 	}
-	if totalCnt > maxLimit && (limit <= 0 || limit > maxLimit) {
+	if int64(totalCnt) > maxLimit && (limit <= 0 || limit > maxLimit) {
 		limit = maxLimit
 	}
+
+	// export data only
+	exportLimit, err := query.Int("export_limit")
+	if query.Contains("export_keys") && err == nil {
+		limit = exportLimit
+	}
+
 	orderBy := jsonutils.GetQueryStringArray(queryDict, "order_by")
 	if len(orderBy) == 0 {
 		colSpec := manager.TableSpec().ColumnSpec("id")
@@ -524,14 +534,14 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		return nil, httperrors.NewGeneralError(err)
 	}
 	if len(retList) != retCount {
-		totalCnt = int64(len(retList))
+		totalCnt = len(retList)
 	}
 	paginate := false
 	if !customizeFilters.IsEmpty() {
 		// query not use Limit and Offset, do manual pagination
 		paginate = true
 	}
-	return calculateListResult(retList, totalCnt, limit, offset, paginate), nil
+	return calculateListResult(retList, int64(totalCnt), limit, offset, paginate), nil
 }
 
 func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64, paginate bool) *modules.ListResult {
@@ -883,7 +893,11 @@ func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient.
 	generateName, _ := dataDict.GetString("generate_name")
 	if len(generateName) > 0 {
 		dataDict.Remove("generate_name")
-		dataDict.Add(jsonutils.NewString(GenerateName(manager, ownerProjId, generateName)), "name")
+		newName, err := GenerateName(manager, ownerProjId, generateName)
+		if err != nil {
+			return nil, err
+		}
+		dataDict.Add(jsonutils.NewString(newName), "name")
 	} else {
 		name, _ := data.GetString("name")
 		if len(name) > 0 {

pkg/cloudcommon/db/fetch.go

@@ -46,7 +46,10 @@ func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query
 	cond := sqlchemy.AND(sqlchemy.Equals(masterField, masterId), sqlchemy.Equals(slaveField, slaveId))
 	q = q.Filter(cond)
 	q = manager.FilterByParams(q, query)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count > 1 {
 		return nil, sqlchemy.ErrDuplicateEntry
 	} else if count == 0 {
@@ -62,7 +65,10 @@ func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query
 func FetchById(manager IModelManager, idStr string) (IModel, error) {
 	q := manager.Query()
 	q = manager.FilterById(q, idStr)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count == 1 {
 		obj, err := NewModelObject(manager)
 		if err != nil {
@@ -88,10 +94,16 @@ func FetchByName(manager IModelManager, userCred mcclient.IIdentityProvider, idS
 	}
 	q := manager.Query()
 	q = manager.FilterByName(q, idStr)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count > 1 {
 		q = manager.FilterByOwner(q, owner)
-		count = q.Count()
+		count, err = q.CountWithError()
+		if err != nil {
+			return nil, err
+		}
 	}
 	if count == 1 {
 		obj, err := NewModelObject(manager)
@@ -133,7 +145,10 @@ func fetchItemById(manager IModelManager, ctx context.Context, userCred mcclient
 		}
 	}
 	q = manager.FilterById(q, idStr)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count == 1 {
 		item, err := NewModelObject(manager)
 		if err != nil {
@@ -161,10 +176,16 @@ func fetchItemByName(manager IModelManager, ctx context.Context, userCred mcclie
 		}
 	}
 	q = manager.FilterByName(q, idStr)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count > 1 {
 		q = manager.FilterByOwner(q, manager.GetOwnerId(userCred))
-		count = q.Count()
+		count, err = q.CountWithError()
+		if err != nil {
+			return nil, err
+		}
 	}
 	if count == 1 {
 		item, err := NewModelObject(manager)

pkg/cloudcommon/db/metadata.go

@@ -147,10 +147,10 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	conditions := []sqlchemy.ICondition{}
 	admin := jsonutils.QueryBoolean(query, "admin", false)
 	for _, resource := range resources {
-		if manager, ok := ResourceMap[resource]; ok {
-			resourceView := manager.Query().SubQuery()
-			prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", manager.Keyword()))
-			field := sqlchemy.CONCAT(manager.Keyword(), prefix, resourceView.Field("id"))
+		if man, ok := ResourceMap[resource]; ok {
+			resourceView := man.Query().SubQuery()
+			prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", man.Keyword()))
+			field := sqlchemy.CONCAT(man.Keyword(), prefix, resourceView.Field("id"))
 			sq := resourceView.Query(field)
 			if !admin || !IsAdminAllowList(userCred, manager) {
 				ownerId := manager.GetOwnerId(userCred)
@@ -171,6 +171,18 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 			q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix))
 		}
 	}
+
+	withConditions := []sqlchemy.ICondition{}
+	for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} {
+		if jsonutils.QueryBoolean(query, args, false) {
+			withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), prefix))
+		}
+	}
+
+	if len(withConditions) > 0 {
+		q = q.Filter(sqlchemy.OR(withConditions...))
+	}
+
 	return q, nil
 }
 
@@ -270,9 +282,6 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 
 	changes := make([]sMetadataChange, 0)
 	for key, value := range store {
-		if strings.HasPrefix(key, SYS_TAG_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, obj, "metadata")) {
-			return nil, httperrors.NewForbiddenError("Ordinary users can't set the tags that begin with an underscore")
-		}
 
 		valStr := stringutils.Interface2String(value)
 		valStrLower := strings.ToLower(valStr)

pkg/cloudcommon/db/models.go

@@ -29,7 +29,6 @@ func RegisterModelManager(modelMan IModelManager) {
 		globalTables = make(map[string]IModelManager)
 	}
 	mustCheckModelManager(modelMan)
-	log.Infof("Register model %s", modelMan.Keyword())
 	globalTables[modelMan.Keyword()] = modelMan
 }
 
@@ -60,8 +59,7 @@ func mustCheckModelManager(modelMan IModelManager) {
 func CheckSync(autoSync bool) bool {
 	log.Infof("Start check database ...")
 	allSqls := make([]string, 0)
-	for modelName, modelMan := range globalTables {
-		log.Infof("# check table of model %s", modelName)
+	for _, modelMan := range globalTables {
 		tableSpec := modelMan.TableSpec()
 		sqls := tableSpec.SyncSQL()
 		for _, sql := range sqls {

pkg/cloudcommon/db/namevalidator.go

@@ -23,13 +23,17 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
-func isNameUnique(manager IModelManager, owner string, name string) bool {
+func isNameUnique(manager IModelManager, owner string, name string) (bool, error) {
 	q := manager.Query()
 	q = manager.FilterByName(q, name)
 	if !consts.IsGlobalVirtualResourceNamespace() {
 		q = manager.FilterByOwner(q, owner)
 	}
-	return q.Count() == 0
+	cnt, err := q.CountWithError()
+	if err != nil {
+		return false, err
+	}
+	return cnt == 0, nil
 }
 
 func NewNameValidator(manager IModelManager, ownerProjId string, name string) error {
@@ -37,13 +41,17 @@ func NewNameValidator(manager IModelManager, ownerProjId string, name string) er
 	if err != nil {
 		return err
 	}
-	if !isNameUnique(manager, ownerProjId, name) {
+	uniq, err := isNameUnique(manager, ownerProjId, name)
+	if err != nil {
+		return err
+	}
+	if !uniq {
 		return httperrors.NewDuplicateNameError("name", name)
 	}
 	return nil
 }
 
-func isAlterNameUnique(model IModel, name string) bool {
+func isAlterNameUnique(model IModel, name string) (bool, error) {
 	manager := model.GetModelManager()
 	q := manager.Query()
 	q = manager.FilterByName(q, name)
@@ -51,7 +59,11 @@ func isAlterNameUnique(model IModel, name string) bool {
 		q = manager.FilterByOwner(q, model.GetOwnerProjectId())
 	}
 	q = manager.FilterByNotId(q, model.GetId())
-	return q.Count() == 0
+	cnt, err := q.CountWithError()
+	if err != nil {
+		return false, err
+	}
+	return cnt == 0, nil
 }
 
 func alterNameValidator(model IModel, name string) error {
@@ -59,13 +71,17 @@ func alterNameValidator(model IModel, name string) error {
 	if err != nil {
 		return err
 	}
-	if !isAlterNameUnique(model, name) {
+	uniq, err := isAlterNameUnique(model, name)
+	if err != nil {
+		return err
+	}
+	if !uniq {
 		return httperrors.NewDuplicateNameError("name", name)
 	}
 	return nil
 }
 
-func GenerateName(manager IModelManager, ownerProjId string, hint string) string {
+func GenerateName(manager IModelManager, ownerProjId string, hint string) (string, error) {
 	_, pattern, patternLen := stringutils.ParseNamePattern(hint)
 	var name string
 	idx := 1
@@ -75,9 +91,15 @@ func GenerateName(manager IModelManager, ownerProjId string, hint string) string
 		name = fmt.Sprintf(pattern, idx)
 		idx += 1
 	}
-	for !isNameUnique(manager, ownerProjId, name) {
+	for {
+		uniq, err := isNameUnique(manager, ownerProjId, name)
+		if err != nil {
+			return "", err
+		}
+		if uniq {
+			return name, nil
+		}
 		name = fmt.Sprintf(pattern, idx)
 		idx += 1
 	}
-	return name
 }

pkg/cloudcommon/db/opslog.go

@@ -32,16 +32,18 @@ import (
 )
 
 const (
-	ACT_CREATE  = "create"
-	ACT_DELETE  = "delete"
-	ACT_UPDATE  = "update"
-	ACT_FETCH   = "fetch"
-	ACT_ENABLE  = "enable"
-	ACT_DISABLE = "disable"
-	ACT_OFFLINE = "offline"
-	ACT_ONLINE  = "online"
-	ACT_ATTACH  = "attach"
-	ACT_DETACH  = "detach"
+	ACT_CREATE      = "create"
+	ACT_DELETE      = "delete"
+	ACT_UPDATE      = "update"
+	ACT_FETCH       = "fetch"
+	ACT_ENABLE      = "enable"
+	ACT_DISABLE     = "disable"
+	ACT_OFFLINE     = "offline"
+	ACT_ONLINE      = "online"
+	ACT_ATTACH      = "attach"
+	ACT_DETACH      = "detach"
+	ACT_ATTACH_FAIL = "attach_fail"
+	ACT_DETACH_FAIL = "detach_fail"
 
 	ACT_SYNC_UPDATE = "sync_update"
 	ACT_SYNC_CREATE = "sync_create"
@@ -49,6 +51,8 @@ const (
 	ACT_START_CREATE_BACKUP  = "start_create_backup"
 	ACT_CREATE_BACKUP        = "create_backup"
 	ACT_CREATE_BACKUP_FAILED = "create_backup_failed"
+	ACT_DELETE_BACKUP        = "delete_backup"
+	ACT_DELETE_BACKUP_FAILED = "delete_backup_failed"
 
 	ACT_UPDATE_STATUS       = "updatestatus"
 	ACT_STARTING            = "starting"
@@ -116,6 +120,9 @@ const (
 	ACT_SET_METADATA = "set_meta"
 	ACT_DEL_METADATA = "del_meta"
 
+	ACT_VM_DEPLOY      = "deploy"
+	ACT_VM_DEPLOY_FAIL = "deploy_fail"
+
 	ACT_REBUILDING_ROOT   = "rebuilding_root"
 	ACT_REBUILD_ROOT      = "rebuild_root"
 	ACT_REBUILD_ROOT_FAIL = "rebuild_root_fail"

pkg/cloudcommon/db/pending_deletable.go

@@ -12,14 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package guestman
+package db
 
 import (
-	"testing"
+	"time"
 )
 
-func TestSKVMGuestInstance_getPid(t *testing.T) {
-	manager := NewGuestManager(nil, "/opt/cloud/workspace/servers")
-	s := NewKVMGuestInstance("05b787e9-b78e-4ebc-8128-04f55d37306f", manager)
-	t.Logf("Guest is ->> %d", s.GetPid())
+type IPendingDeletable interface {
+	GetPendingDeleted() bool
+	GetPendingDeletedAt() time.Time
 }

pkg/cloudcommon/db/standalone.go

@@ -99,7 +99,10 @@ func (manager *SStandaloneResourceBaseManager) FetchByIdOrName(userCred mcclient
 
 func (manager *SStandaloneResourceBaseManager) FetchByExternalId(idStr string) (IStandaloneModel, error) {
 	q := manager.Query().Equals("external_id", idStr)
-	count := q.Count()
+	count, err := q.CountWithError()
+	if err != nil {
+		return nil, err
+	}
 	if count == 1 {
 		obj, err := NewModelObject(manager)
 		if err != nil {
@@ -153,7 +156,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex
 	}
 
 	if len(tags) > 0 {
-		metadataView := Metadata.Query("id")
+		metadataView := Metadata.Query()
 		idx := 0
 		for k, v := range tags {
 			if idx == 0 {
@@ -171,25 +174,26 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex
 			}
 			idx++
 		}
-		metadataView = metadataView.Filter(sqlchemy.Like(metadataView.Field("id"), manager.Keyword()+"::%")).Distinct()
-		resourceIds := []string{}
-		rows, err := metadataView.Rows()
-		if err != nil {
-			log.Errorf("query metadata ids error: %v", err)
-			return nil, err
-		}
-		defer rows.Close()
-		for rows.Next() {
-			var metadataID string
-			err = rows.Scan(&metadataID)
-			if err != nil {
-				log.Errorf("get metadata id scan error: %v", err)
-				return nil, err
-			}
-			resourceIds = append(resourceIds, strings.TrimLeft(metadataID, manager.Keyword()+"::"))
-		}
-		q = q.Filter(sqlchemy.In(q.Field("id"), resourceIds))
+		metadatas := metadataView.SubQuery()
+		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
+		metadataSQ := metadatas.Query(
+			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
+		)
+		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("id"), manager.Keyword()+"::%")).Distinct()
+		q = q.Filter(sqlchemy.In(q.Field("id"), sq))
 	}
+
+	if withoutUserMeta, _ := query.Bool("without_user_meta"); withoutUserMeta {
+		metadatas := Metadata.Query().SubQuery()
+		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
+		metadataSQ := metadatas.Query(
+			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
+		)
+		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("key"), USER_TAG_PREFIX+"%")).Distinct()
+
+		q.Filter(sqlchemy.NotIn(q.Field("id"), sq))
+	}
+
 	return q, nil
 }
 
@@ -227,18 +231,10 @@ func (model *SStandaloneResourceBase) GetMetadataJson(key string, userCred mccli
 }
 
 func (model *SStandaloneResourceBase) SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error {
-	if Metadata.IsSystemAdminKey(key) && !IsAdminAllowPerform(userCred, model, "metadata") {
-		return httperrors.NewNotSufficientPrivilegeError("cannot set system key")
-	}
 	return Metadata.SetValue(ctx, model, key, value, userCred)
 }
 
 func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
-	for k := range dictstore {
-		if Metadata.IsSystemAdminKey(k) && !IsAdminAllowPerform(userCred, model, "metadata") {
-			return httperrors.NewNotSufficientPrivilegeError("not allow to set system key %s", k)
-		}
-	}
 	return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
 }
 
@@ -290,6 +286,10 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC
 	}
 	dictStore := make(map[string]interface{})
 	for k, v := range dictMap {
+		// 已双下滑线开头的metadata是系统内置，普通用户不可添加，只能查看
+		if strings.HasPrefix(k, SYS_TAG_PREFIX) && (userCred == nil || !IsAdminAllowPerform(userCred, model, "metadata")) {
+			return nil, httperrors.NewForbiddenError("not allow to set system key, please remove the underscore at the beginning")
+		}
 		dictStore[k], _ = v.GetString()
 	}
 	err = model.SetAllMetadata(ctx, dictStore, userCred)
@@ -311,7 +311,7 @@ func (model *SStandaloneResourceBase) PerformUserMetadata(ctx context.Context, u
 	}
 	dictStore := make(map[string]interface{})
 	for k, v := range dictMap {
-		dictStore["user:"+k], _ = v.GetString()
+		dictStore[USER_TAG_PREFIX+k], _ = v.GetString()
 	}
 	err = model.SetUserMetadataValues(ctx, dictStore, userCred)
 	return nil, err

pkg/cloudcommon/db/taskman/tasks.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"runtime/debug"
 	"strings"
 	"time"
 
@@ -75,9 +76,9 @@ type STask struct {
 	ObjName  string                   `width:"128" charset:"utf8" nullable:"false" list:"user"`               //  Column(VARCHAR(128, charset='utf8'), nullable=False)
 	ObjId    string                   `width:"128" charset:"ascii" nullable:"false" list:"user" index:"true"` // Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
 	TaskName string                   `width:"64" charset:"ascii" nullable:"false" list:"user"`               // Column(VARCHAR(64, charset='ascii'), nullable=False)
-	UserCred mcclient.TokenCredential `width:"1024" charset:"ascii" nullable:"false" get:"user"`              // Column(VARCHAR(1024, charset='ascii'), nullable=False)
+	UserCred mcclient.TokenCredential `width:"1024" charset:"utf8" nullable:"false" get:"user"`               // Column(VARCHAR(1024, charset='ascii'), nullable=False)
 	// OwnerCred string `width:"512" charset:"ascii" nullable:"true"` // Column(VARCHAR(512, charset='ascii'), nullable=True)
-	Params *jsonutils.JSONDict `charset:"ascii" length:"medium" nullable:"false" get:"user"` // Column(MEDIUMTEXT(charset='ascii'), nullable=False)
+	Params *jsonutils.JSONDict `charset:"utf8" length:"medium" nullable:"false" get:"user"` // Column(MEDIUMTEXT(charset='ascii'), nullable=False)
 
 	Stage string `width:"64" charset:"ascii" nullable:"false" default:"on_init" list:"user"` // Column(VARCHAR(64, charset='ascii'), nullable=False, default='on_init')
 
@@ -453,6 +454,21 @@ func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject,
 		return
 	}
 
+	defer func() {
+		if r := recover(); r != nil {
+			// call set stage failed, should not call task.SetStageFailed
+			// func SetStageFailed may be overloading
+			log.Errorf("Task %s PANIC on stage %s: %v \n%s", task.TaskName, stageName, r, debug.Stack())
+			SetStageFailedFuncValue := taskValue.MethodByName("SetStageFailed")
+			SetStageFailedFuncValue.Call(
+				[]reflect.Value{
+					reflect.ValueOf(ctx),
+					reflect.ValueOf(fmt.Sprintf("%v", r)),
+				},
+			)
+		}
+	}()
+
 	log.Debugf("Call %s %s %#v", task.TaskName, stageName, params)
 	funcValue.Call(params)
 
@@ -753,9 +769,8 @@ func (manager *STaskManager) QueryTasksOfObject(obj db.IStandaloneModel, since t
 	// subq1 and subq2 do not intersect for the fact that they have
 	// different condition on tasks_tbl.obj_id field
 	uq := sqlchemy.Union(subq1, subq2)
-	uq = uq.Desc("created_at")
 
-	q := uq.SubQuery().Query()
+	q := uq.Query().Desc("created_at")
 	return q
 }
 

pkg/cloudcommon/db/update.go

@@ -22,11 +22,11 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 )
 
-func Update(model IModel, updateFunc func() error) (map[string]sqlchemy.SUpdateDiff, error) {
+func Update(model IModel, updateFunc func() error) (sqlchemy.UpdateDiffs, error) {
 	return model.GetModelManager().TableSpec().Update(model, updateFunc)
 }
 
-func UpdateWithLock(ctx context.Context, model IModel, updateFunc func() error) (map[string]sqlchemy.SUpdateDiff, error) {
+func UpdateWithLock(ctx context.Context, model IModel, updateFunc func() error) (sqlchemy.UpdateDiffs, error) {
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 

pkg/cloudcommon/db/virtualresource.go

@@ -251,7 +251,11 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC
 	q := model.GetModelManager().Query().Equals("name", model.GetName())
 	q = q.Equals("tenant_id", tobj.GetId())
 	q = q.NotEquals("id", model.GetId())
-	if q.Count() > 0 {
+	cnt, err := q.CountWithError()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("check name duplication error: %s", err)
+	}
+	if cnt > 0 {
 		return nil, httperrors.NewDuplicateNameError("name", model.GetName())
 	}
 	former, _ := TenantCacheManager.FetchTenantById(ctx, model.ProjectId)
@@ -260,7 +264,7 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC
 		formerObj := NewTenant(model.ProjectId, "unknown")
 		former = &formerObj
 	}
-	_, err := Update(model, func() error {
+	_, err = Update(model, func() error {
 		model.ProjectId = tobj.GetId()
 		model.ProjectSrc = string(PROJECT_SOURCE_LOCAL)
 		return nil
@@ -353,8 +357,12 @@ func (model *SVirtualResourceBase) MarkCancelPendingDelete(ctx context.Context,
 	lockman.LockClass(ctx, model.GetModelManager(), ownerProjId)
 	defer lockman.ReleaseClass(ctx, model.GetModelManager(), ownerProjId)
 
+	newName, err := GenerateName(model.GetModelManager(), ownerProjId, model.Name)
+	if err != nil {
+		return err
+	}
 	diff, err := Update(model, func() error {
-		model.Name = GenerateName(model.GetModelManager(), ownerProjId, model.Name)
+		model.Name = newName
 		model.PendingDeleted = false
 		model.PendingDeletedAt = time.Time{}
 		return nil
@@ -391,3 +399,13 @@ func (model *SVirtualResourceBase) SyncCloudProjectId(userCred mcclient.TokenCre
 		}
 	}
 }
+
+// GetPendingDeleted implements IPendingDeltable
+func (model *SVirtualResourceBase) GetPendingDeleted() bool {
+	return model.PendingDeleted
+}
+
+// GetPendingDeletedAt implements IPendingDeltable
+func (model *SVirtualResourceBase) GetPendingDeletedAt() time.Time {
+	return model.PendingDeletedAt
+}

pkg/cloudcommon/options/options.go

@@ -16,16 +16,20 @@ package options
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 
 	"yunion.io/x/log"
+	"yunion.io/x/log/hooks"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 type CommonOptions struct {
@@ -34,6 +38,7 @@ type CommonOptions struct {
 
 	LogLevel        string `help:"log level" default:"info" choices:"debug|info|warn|error"`
 	LogVerboseLevel int    `help:"log verbosity level" default:"0"`
+	LogFilePrefix   string `help:"prefix of log files"`
 
 	Region             string   `help:"Region name or ID" alias:"auth-region"`
 	AuthURL            string   `help:"Keystone auth URL" alias:"auth-uri"`
@@ -120,7 +125,6 @@ func ParseOptions(optStruct interface{}, args []string, configFileName string, s
 	if len(optionsRef.Config) == 0 {
 		for _, p := range []string{"./etc", "/etc/yunion"} {
 			confTmp := path.Join(p, configFileName)
-			log.Infof(confTmp)
 			if _, err := os.Stat(confTmp); err == nil {
 				optionsRef.Config = confTmp
 				break
@@ -148,6 +152,28 @@ func ParseOptions(optStruct interface{}, args []string, configFileName string, s
 	if err != nil {
 		log.Fatalf("Set log level %q: %v", optionsRef.LogLevel, err)
 	}
+	if optionsRef.LogFilePrefix != "" {
+		dir, name := filepath.Split(optionsRef.LogFilePrefix)
+		h := &hooks.LogFileRotateHook{
+			RotateNum:  10,
+			RotateSize: 100 * 1024 * 1024,
+			LogFileHook: hooks.LogFileHook{
+				FileDir:  dir,
+				FileName: name,
+			},
+		}
+		h.Init()
+		log.DisableColors()
+		log.Logger().AddHook(h)
+		log.Logger().Out = ioutil.Discard
+		atexit.Register(atexit.ExitHandler{
+			Prio:   atexit.PRIO_LOG_CLOSE,
+			Reason: "deinit log rotate hook",
+			Func: func(atexit.ExitHandler) {
+				h.DeInit()
+			},
+		})
+	}
 
 	log.V(10).Debugf("Parsed options: %#v", optStruct)
 

pkg/cloudcommon/service/services.go

@@ -14,4 +14,67 @@
 
 package service
 
-type SServiceBase struct{}
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type IService interface {
+	InitService()
+	RunService()
+	OnExitService()
+}
+
+type SServiceBase struct {
+	Service IService
+
+	O *options.CommonOptions
+}
+
+func (s *SServiceBase) StartService() {
+	defer s.Service.OnExitService()
+	defer s.RemovePid()
+
+	s.Service.InitService()
+	if err := s.CreatePid(); err != nil {
+		log.Fatalln(err)
+	}
+
+	s.Service.RunService()
+}
+
+func (s *SServiceBase) CreatePid() error {
+	if s.O == nil || len(s.O.PidFile) == 0 {
+		return nil
+	}
+	absPath, err := filepath.Abs(s.O.PidFile)
+	if err != nil {
+		return fmt.Errorf("Get pidfile %s absolute path failed: %s", s.O.PidFile, err)
+	}
+	s.O.PidFile = absPath
+	pidDir := filepath.Dir(s.O.PidFile)
+	if !fileutils2.Exists(pidDir) {
+		output, err := procutils.NewCommand("mkdir", "-p", pidDir).Run()
+		if err != nil {
+			return fmt.Errorf("Make pid dir %s failed: %s", pidDir, output)
+		}
+	}
+	err = fileutils2.FilePutContents(s.O.PidFile, strconv.Itoa(os.Getpid()), false)
+	if err != nil {
+		return fmt.Errorf("Write pidfile %s failed: %s", s.O.PidFile, err)
+	}
+	return nil
+}
+
+func (s *SServiceBase) RemovePid() {
+	if s.O != nil && len(s.O.PidFile) > 0 && fileutils2.Exists(s.O.PidFile) {
+		os.Remove(s.O.PidFile)
+	}
+}

pkg/cloudcommon/types/types.go

@@ -39,10 +39,11 @@ func (info *SDMISystemInfo) ToIPMISystemInfo() *SIPMISystemInfo {
 }
 
 type SCPUInfo struct {
-	Count int    `json:"count"`
-	Model string `json:"desc"`
-	Freq  int    `json:"freq"`
-	Cache int    `json:"cache"`
+	Count     int    `json:"count"`
+	Model     string `json:"desc"`
+	Freq      int    `json:"freq"`
+	Cache     int    `json:"cache"`
+	Microcode string `json:"microcode"`
 }
 
 type SDMICPUInfo struct {

pkg/cloudcommon/validators/errors.go

@@ -98,7 +98,7 @@ func newInvalidStructError(key string, err error) error {
 	params := []interface{}{key}
 	jsonClientErr, ok := err.(*httputils.JSONClientError)
 	if ok {
-		errFmt += jsonClientErr.Data.Id
+		errFmt += httperrors.MsgTmplToFmt(jsonClientErr.Data.Id)
 		for _, f := range jsonClientErr.Data.Fields {
 			params = append(params, f)
 		}
@@ -113,7 +113,7 @@ func newModelManagerError(modelKeyword string) error {
 func newModelNotFoundError(modelKeyword, idOrName string, err error) error {
 	errFmt := "cannot find %q with id/name %q"
 	params := []interface{}{modelKeyword, idOrName}
-	if err != sql.ErrNoRows {
+	if err != nil && err != sql.ErrNoRows {
 		errFmt += ": %s"
 		params = append(params, err.Error())
 	}

pkg/cloudcommon/validators/validators.go

@@ -375,12 +375,13 @@ func NewNonNegativeValidator(key string) *ValidatorRange {
 
 type ValidatorModelIdOrName struct {
 	Validator
-	ModelKeyword string
-	ProjectId    string
-	UserId       string
-	ModelManager db.IModelManager
-	Model        db.IModel
-	modelIdKey   string
+	ModelKeyword     string
+	ProjectId        string
+	UserId           string
+	ModelManager     db.IModelManager
+	Model            db.IModel
+	modelIdKey       string
+	noPendingDeleted bool
 }
 
 func (v *ValidatorModelIdOrName) GetProjectId() string {
@@ -401,10 +402,11 @@ func (v *ValidatorModelIdOrName) getValue() interface{} {
 
 func NewModelIdOrNameValidator(key string, modelKeyword string, projectId string) *ValidatorModelIdOrName {
 	v := &ValidatorModelIdOrName{
-		Validator:    Validator{Key: key},
-		ProjectId:    projectId,
-		ModelKeyword: modelKeyword,
-		modelIdKey:   key + "_id",
+		Validator:        Validator{Key: key},
+		ProjectId:        projectId,
+		ModelKeyword:     modelKeyword,
+		modelIdKey:       key + "_id",
+		noPendingDeleted: true,
 	}
 	v.parent = v
 	return v
@@ -415,6 +417,12 @@ func (v *ValidatorModelIdOrName) ModelIdKey(modelIdKey string) *ValidatorModelId
 	return v
 }
 
+// AllowPendingDeleted allows the to-be-validated id or name to be of a pending deleted model
+func (v *ValidatorModelIdOrName) AllowPendingDeleted(b bool) *ValidatorModelIdOrName {
+	v.noPendingDeleted = !b
+	return v
+}
+
 func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error {
 	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
 		return err
@@ -433,6 +441,11 @@ func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error {
 	if err != nil {
 		return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
 	}
+	if v.noPendingDeleted {
+		if pd, ok := model.(db.IPendingDeletable); ok && pd.GetPendingDeleted() {
+			return newModelNotFoundError(v.ModelKeyword, modelIdOrName, nil)
+		}
+	}
 	v.Model = model
 	return nil
 }

pkg/cloudprovider/consts.go

@@ -34,3 +34,4 @@ var ErrTimeout = errors.New("timeout")
 var ErrNotImplemented = errors.New("Not implemented")
 var ErrNotSupported = errors.New("Not supported")
 var ErrInvalidProvider = errors.New("Invalid provider")
+var ErrNoBalancePermission = errors.New("No balance permission")

pkg/cloudprovider/images.go

@@ -71,3 +71,14 @@ func CloudImage2Image(image ICloudImage) SImage {
 		Status:    image.GetImageStatus(),
 	}
 }
+
+type SImageCreateOption struct {
+	ImageId        string
+	ExternalId     string
+	ImageName      string
+	OsType         string
+	OsArch         string
+	OsDistribution string
+	OsVersion      string
+	OsFullVersion  string
+}

pkg/cloudprovider/instance.go

@@ -15,6 +15,8 @@
 package cloudprovider
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/util/osprofile"
 
@@ -53,9 +55,9 @@ type SManagedVMCreateConfig struct {
 	PublicKey           string
 	ExternalSecgroupId  string
 	ExternalSecgroupIds []string
+	Account             string
 	Password            string
 	UserData            string
-	UserDataType        string
 
 	BillingCycle *billing.SBillingCycle
 }
@@ -69,15 +71,11 @@ func (vmConfig *SManagedVMCreateConfig) GetConfig(config *jsonutils.JSONDict) er
 	}
 	//目前所写的userData格式仅支持Linux
 	if vmConfig.OsType == osprofile.OS_TYPE_LINUX {
-		if userDataType, _ := config.GetString("user_data_type"); len(userDataType) > 0 {
-			vmConfig.UserDataType = userDataType
-		}
-
 		adminPublicKey, _ := config.GetString("admin_public_key")
 		projectPublicKey, _ := config.GetString("project_public_key")
 		oUserData, _ := config.GetString("user_data")
 
-		vmConfig.UserData = generateUserData(adminPublicKey, projectPublicKey, oUserData, vmConfig.UserDataType)
+		vmConfig.UserData = generateUserData(adminPublicKey, projectPublicKey, oUserData)
 	}
 
 	resetPassword := jsonutils.QueryBoolean(config, "reset_password", false)
@@ -88,11 +86,11 @@ func (vmConfig *SManagedVMCreateConfig) GetConfig(config *jsonutils.JSONDict) er
 	return nil
 }
 
-func generateUserData(adminPublicKey, projectPublicKey, oUserData string, userDataType string) string {
+func generateUserData(adminPublicKey, projectPublicKey, oUserData string) string {
 	var oCloudConfig *cloudinit.SCloudConfig
 
 	if len(oUserData) > 0 {
-		oCloudConfig, _ = cloudinit.ParseUserDataBase64(oUserData)
+		oCloudConfig, _ = cloudinit.ParseUserData(oUserData)
 	}
 
 	ansibleUser := cloudinit.NewUser(ansible.PUBLIC_CLOUD_ANSIBLE_USER)
@@ -111,9 +109,37 @@ func generateUserData(adminPublicKey, projectPublicKey, oUserData string, userDa
 		cloudConfig.Merge(oCloudConfig)
 	}
 
-	if userDataType == CLOUD_SHELL {
-		return cloudConfig.UserDataScriptBase64()
+	return cloudConfig.UserData()
+}
+
+func (vmConfig *SManagedVMCreateConfig) InjectPasswordByCloudInit() error {
+	if vmConfig.OsType != osprofile.OS_TYPE_LINUX {
+		return fmt.Errorf("Only support inject Linux password, current osType is %s", vmConfig.OsType)
+	}
+	loginUser := cloudinit.NewUser(vmConfig.Account)
+	loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD)
+	if len(vmConfig.PublicKey) > 0 {
+		loginUser.SshKey(vmConfig.PublicKey)
+	}
+	if len(vmConfig.Password) > 0 {
+		loginUser.Password(vmConfig.Password)
 	}
 
-	return cloudConfig.UserDataBase64()
+	cloudconfig := cloudinit.SCloudConfig{
+		DisableRoot: 0,
+		SshPwauth:   1,
+		Users: []cloudinit.SUser{
+			loginUser,
+		},
+	}
+
+	if len(vmConfig.UserData) > 0 {
+		oCloudConfig, err := cloudinit.ParseUserData(vmConfig.UserData)
+		if err != nil {
+			return err
+		}
+		cloudconfig.Merge(oCloudConfig)
+	}
+	vmConfig.UserData = cloudconfig.UserData()
+	return nil
 }

pkg/cloudprovider/loadbalancerlistener.go

@@ -31,6 +31,9 @@ type SLoadbalancerListener struct {
 	Description             string
 	EstablishedTimeout      int
 
+	HealthCheckReq string
+	HealthCheckExp string
+
 	HealthCheck         string
 	HealthCheckTimeout  int
 	HealthCheckDomain   string

pkg/cloudprovider/resources.go

@@ -45,6 +45,7 @@ type IVirtualResource interface {
 
 type IBillingResource interface {
 	GetBillingType() string
+	GetCreatedAt() time.Time
 	GetExpiredAt() time.Time
 }
 
@@ -136,13 +137,11 @@ type ICloudStoragecache interface {
 
 	GetPath() string
 
-	GetManagerId() string
-
 	CreateIImage(snapshotId, imageName, osType, imageDesc string) (ICloudImage, error)
 
 	DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error)
 
-	UploadImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist, osVersion string, extId string, isForce bool) (string, error)
+	UploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *SImageCreateOption, isForce bool) (string, error)
 }
 
 type ICloudStorage interface {
@@ -159,8 +158,6 @@ type ICloudStorage interface {
 	GetStorageConf() jsonutils.JSONObject
 	GetEnabled() bool
 
-	GetManagerId() string
-
 	CreateIDisk(name string, sizeGb int, desc string) (ICloudDisk, error)
 
 	GetIDiskById(idStr string) (ICloudDisk, error)
@@ -199,8 +196,6 @@ type ICloudHost interface {
 	GetIsMaintenance() bool
 	GetVersion() string
 
-	GetManagerId() string
-
 	CreateVM(desc *SManagedVMCreateConfig) (ICloudVM, error)
 	GetIHostNics() ([]ICloudHostNetInterface, error)
 }
@@ -287,8 +282,6 @@ type ICloudEIP interface {
 
 	GetInternetChargeType() string
 
-	GetManagerId() string
-
 	Delete() error
 
 	Associate(instanceId string) error
@@ -308,7 +301,6 @@ type ICloudSecurityGroup interface {
 
 type ICloudRouteTable interface {
 	ICloudResource
-	GetManagerId() string
 
 	GetDescription() string
 	GetRegionId() string
@@ -362,7 +354,7 @@ type ICloudSnapshot interface {
 	ICloudResource
 	IVirtualResource
 
-	GetSize() int32
+	GetSizeMb() int32
 	GetDiskId() string
 	GetDiskType() string
 	Delete() error
@@ -379,8 +371,6 @@ type ICloudVpc interface {
 	GetISecurityGroups() ([]ICloudSecurityGroup, error)
 	GetIRouteTables() ([]ICloudRouteTable, error)
 
-	GetManagerId() string
-
 	Delete() error
 
 	GetIWireById(wireId string) (ICloudWire, error)
@@ -526,6 +516,7 @@ type ICloudLoadbalancerBackendGroup interface {
 	IsDefault() bool
 	GetType() string
 	GetILoadbalancerBackends() ([]ICloudLoadbalancerBackend, error)
+	GetILoadbalancerBackendById(backendId string) (ICloudLoadbalancerBackend, error)
 	AddBackendServer(serverId string, weight int, port int) (ICloudLoadbalancerBackend, error)
 	RemoveBackendServer(serverId string, weight int, port int) error
 
@@ -542,6 +533,7 @@ type ICloudLoadbalancerBackend interface {
 	GetBackendType() string
 	GetBackendRole() string
 	GetBackendId() string
+	SyncConf(port, weight int) error
 }
 
 type ICloudLoadbalancerCertificate interface {

pkg/compute/baremetal/diskconfig.go

@@ -82,8 +82,18 @@ func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*Baremetal
 	selected = make([]*BaremetalStorage, 0)
 	rest = make([]*BaremetalStorage, 0)
 	idx := 0
+	curAdapter := 0
+	adapterChange := false
 
 	for _, storage := range storages {
+		if storage.Adapter != curAdapter {
+			adapterChange = true
+			curAdapter = storage.Adapter
+		}
+		if adapterChange {
+			idx = 0
+			adapterChange = false
+		}
 		if storage.Index == 0 {
 			storage.Index = int64(idx)
 		}
@@ -535,10 +545,11 @@ func GetDiskSpecV2(storages []*BaremetalStorage) api.DiskDriverSpec {
 }
 
 type DiskConfiguration struct {
-	Driver  string
-	Adapter int
-	Block   int64
-	Size    int64
+	Driver     string
+	Adapter    int
+	RaidConfig string
+	Block      int64
+	Size       int64
 }
 
 func GetDiskConfigurations(layouts []Layout) []DiskConfiguration {
@@ -550,15 +561,33 @@ func GetDiskConfigurations(layouts []Layout) []DiskConfiguration {
 		raidConf := rr.Conf.Conf
 		if raidConf == DISK_CONF_NONE {
 			for _, d := range rr.Disks {
-				disks = append(disks, DiskConfiguration{Driver: driver, Adapter: adapter, Block: block, Size: d.Size})
+				disks = append(disks, DiskConfiguration{
+					Driver:     driver,
+					Adapter:    adapter,
+					RaidConfig: raidConf,
+					Block:      block,
+					Size:       d.Size,
+				})
 			}
 		} else {
 			if len(rr.Conf.Size) != 0 {
 				for _, sz := range rr.Conf.Size {
-					disks = append(disks, DiskConfiguration{Driver: driver, Adapter: adapter, Block: block, Size: sz})
+					disks = append(disks, DiskConfiguration{
+						Driver:     driver,
+						Adapter:    adapter,
+						RaidConfig: raidConf,
+						Block:      block,
+						Size:       sz,
+					})
 				}
 			} else {
-				disks = append(disks, DiskConfiguration{Driver: driver, Adapter: adapter, Block: block, Size: rr.Size})
+				disks = append(disks, DiskConfiguration{
+					Driver:     driver,
+					Adapter:    adapter,
+					RaidConfig: raidConf,
+					Block:      block,
+					Size:       rr.Size,
+				})
 			}
 		}
 	}
@@ -592,3 +621,36 @@ func GroupLayoutResultsByDriverAdapter(layouts []Layout) []*DriverAdapterDiskCon
 	}
 	return ret
 }
+
+func ValidateDiskConfigs(confs []*api.BaremetalDiskConfig) error {
+	if len(confs) == 0 {
+		return nil
+	}
+	for idx, conf := range confs {
+		if conf.Conf != DISK_CONF_NONE {
+			// raid validate
+			if idx > 0 {
+				preConf := confs[idx-1]
+				if preConf.Conf == DISK_CONF_NONE {
+					return fmt.Errorf("Raid config %d must before none raid config", idx)
+				}
+			}
+		} else {
+			// none raid validate
+			if idx+1 == len(confs) {
+				return nil
+			}
+			restConfs := confs[idx+1:]
+			hasRaidConf := false
+			for _, restConf := range restConfs {
+				if restConf.Conf != DISK_CONF_NONE {
+					hasRaidConf = true
+				}
+			}
+			if hasRaidConf {
+				return fmt.Errorf("Raid config after none raid config %d", idx)
+			}
+		}
+	}
+	return nil
+}

pkg/compute/baremetal/diskconfig_test.go

@@ -19,7 +19,6 @@ import (
 	"reflect"
 	"testing"
 
-	"yunion.io/x/log"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 )
 
@@ -203,7 +202,7 @@ var (
 			Adapter: 2,
 		},
 
-		// 12-13 disk on adapter0
+		// 0-1 disk on adapter0
 		{
 			Driver:  DISK_DRIVER_MEGARAID,
 			Rotate:  true,
@@ -295,7 +294,7 @@ var (
 
 func TestCalculateLayout(t *testing.T) {
 	confs, err := NewBaremetalDiskConfigs(
-		"[12-13]:raid1:(100g,32g,):adapter0",
+		"[0-1]:raid1:(100g,32g,):adapter0",
 		"6:raid5:adapter2",
 		"6:raid5:adapter2",
 	)
@@ -311,13 +310,13 @@ func TestCalculateLayout(t *testing.T) {
         "rotate": true,
         "driver": "MegaRaid",
         "size": 953344,
-        "index": 12
+        "index": 0
       },
       {
         "rotate": true,
         "driver": "MegaRaid",
         "size": 953344,
-        "index": 13
+        "index": 1
       }
     ],
     "conf": {
@@ -325,8 +324,8 @@ func TestCalculateLayout(t *testing.T) {
       "conf": "raid1",
       "count": 0,
       "range": [
-        12,
-        13
+        0,
+        1
       ],
       "splits": "100g,32g,",
       "size": [
@@ -472,7 +471,7 @@ func TestCalculateLayout(t *testing.T) {
 
 func TestCheckDisksAllocable(t *testing.T) {
 	confs, err := NewBaremetalDiskConfigs(
-		"[12-13]:raid1:(100g,32g,):adapter0",
+		"[0-1]:raid1:(100g,32g,):adapter0",
 		"6:raid5:adapter2",
 		"6:raid5:adapter2",
 	)
@@ -491,10 +490,6 @@ func TestCheckDisksAllocable(t *testing.T) {
 		t.Fatalf("Calculate bitmain layout err: %v", err)
 	}
 
-	log.Debugf("defaultLayout: %s", defaultLayout)
-	log.Debugf("layout: %s", layout)
-	log.Debugf("Bitmain layout: %s", bitmainLayout)
-
 	tdiskDefault := []*Disk{
 		{Size: -1},
 		{Size: -1},
@@ -949,11 +944,10 @@ func TestStorageLoad(t *testing.T) {
 		t.Fatalf("NewDiskConfigs err: %v", err)
 	}
 	json.Unmarshal([]byte(testStorages2), &ss)
-	layout, err := CalculateLayout(confs, ss)
+	_, err = CalculateLayout(confs, ss)
 	if err != nil {
 		t.Fatalf("CalculateLayout err: %v", err)
 	}
-	log.Debugf("layout: %s", layout)
 }
 
 func TestCalculateSize(t *testing.T) {
@@ -1076,3 +1070,179 @@ func TestGetDiskSpecs(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateDiskConfigs(t *testing.T) {
+	tests := []struct {
+		name    string
+		confs   []*api.BaremetalDiskConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty",
+			confs:   []*api.BaremetalDiskConfig{},
+			wantErr: false,
+		},
+		{
+			name: "none_raid",
+			confs: []*api.BaremetalDiskConfig{
+				{Conf: DISK_CONF_NONE},
+				{Conf: DISK_CONF_RAID0},
+			},
+			wantErr: true,
+		},
+		{
+			name: "raid_none_raid",
+			confs: []*api.BaremetalDiskConfig{
+				{Conf: DISK_CONF_RAID0},
+				{Conf: DISK_CONF_NONE},
+				{Conf: DISK_CONF_RAID5},
+			},
+			wantErr: true,
+		},
+		{
+			name: "nones",
+			confs: []*api.BaremetalDiskConfig{
+				{Conf: DISK_CONF_NONE},
+				{Conf: DISK_CONF_NONE},
+			},
+			wantErr: false,
+		},
+		{
+			name: "raids_nones",
+			confs: []*api.BaremetalDiskConfig{
+				{Conf: DISK_CONF_RAID0},
+				{Conf: DISK_CONF_RAID5},
+				{Conf: DISK_CONF_NONE},
+				{Conf: DISK_CONF_NONE},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := ValidateDiskConfigs(tt.confs); (err != nil) != tt.wantErr {
+				t.Errorf("ValidateDiskConfigs() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestRetrieveStorages(t *testing.T) {
+	type args struct {
+		diskConfig *api.BaremetalDiskConfig
+		storages   []*BaremetalStorage
+	}
+	s1 := &BaremetalStorage{
+		Size:    1,
+		Driver:  "MegaRaid",
+		Adapter: 0,
+	}
+	s2 := &BaremetalStorage{
+		Size:    1,
+		Driver:  "MegaRaid",
+		Adapter: 0,
+	}
+	s3 := &BaremetalStorage{
+		Size:    2,
+		Driver:  "MegaRaid",
+		Adapter: 0,
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantSelected []*BaremetalStorage
+		wantRest     []*BaremetalStorage
+	}{
+		{
+			name: "select all",
+			args: args{
+				diskConfig: &api.BaremetalDiskConfig{
+					Driver: "MegaRaid",
+					Count:  0,
+					Type:   api.DISK_TYPE_HYBRID,
+				},
+				storages: []*BaremetalStorage{s1, s2, s3},
+			},
+			wantSelected: []*BaremetalStorage{s1, s2, s3},
+			wantRest:     []*BaremetalStorage{},
+		},
+		{
+			name: "select count 2",
+			args: args{
+				diskConfig: &api.BaremetalDiskConfig{
+					Driver: "MegaRaid",
+					Count:  2,
+					Type:   api.DISK_TYPE_HYBRID,
+				},
+				storages: []*BaremetalStorage{s1, s2, s3},
+			},
+			wantSelected: []*BaremetalStorage{s1, s2},
+			wantRest:     []*BaremetalStorage{s3},
+		},
+		{
+			name: "select by range",
+			args: args{
+				diskConfig: &api.BaremetalDiskConfig{
+					Driver: "MegaRaid",
+					Range:  []int64{1, 2},
+					Type:   api.DISK_TYPE_HYBRID,
+				},
+				storages: []*BaremetalStorage{s1, s2, s3},
+			},
+			wantSelected: []*BaremetalStorage{s2, s3},
+			wantRest:     []*BaremetalStorage{s1},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotSelected, gotRest := RetrieveStorages(tt.args.diskConfig, tt.args.storages)
+			if !reflect.DeepEqual(gotSelected, tt.wantSelected) {
+				t.Errorf("RetrieveStorages() gotSelected = %v, want %v", gotSelected, tt.wantSelected)
+			}
+			if !reflect.DeepEqual(gotRest, tt.wantRest) {
+				t.Errorf("RetrieveStorages() gotRest = %v, want %v", gotRest, tt.wantRest)
+			}
+		})
+	}
+}
+
+func TestGetSplitSizes(t *testing.T) {
+	type args struct {
+		size      int64
+		splitConf string
+	}
+	var totalSize int64 = 22889472
+	size10p := int64(float64(totalSize) * 0.1)
+	tests := []struct {
+		name string
+		args args
+		want []int64
+	}{
+		{
+			name: "split gb",
+			args: args{
+				size:      totalSize,
+				splitConf: "60g,",
+			},
+			want: []int64{61440, totalSize - 61440},
+		},
+		{
+			name: "split by percent",
+			args: args{
+				size:      totalSize,
+				splitConf: "10%,",
+			},
+			want: []int64{
+				size10p,
+				totalSize - size10p,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := GetSplitSizes(tt.args.size, tt.args.splitConf); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetSplitSizes() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/compute/guestdrivers/aliyun.go

@@ -42,6 +42,10 @@ func (self *SAliyunGuestDriver) GetHypervisor() string {
 	return api.HYPERVISOR_ALIYUN
 }
 
+func (self *SAliyunGuestDriver) GetProvider() string {
+	return api.CLOUD_PROVIDER_ALIYUN
+}
+
 func (self *SAliyunGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_CLOUD_EFFICIENCY
 }
@@ -128,6 +132,9 @@ func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred
 			}
 		}
 	}
+	if input.EipBw > 100 {
+		return nil, httperrors.NewInputParameterError("%s requires that the eip bandwidth must be less than 100Mbps", self.GetHypervisor())
+	}
 	return input, nil
 }
 
@@ -147,160 +154,6 @@ func (self *SAliyunGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManag
 	return userName
 }
 
-/* func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
-	config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams())
-	if err != nil {
-		log.Errorf("GetDeployConfigOnHost error: %v", err)
-		return err
-	}
-	log.Debugf("RequestDeployGuestOnHost: %s", config)
-
-	desc := cloudprovider.SManagedVMCreateConfig{}
-	if err := desc.GetConfig(config); err != nil {
-		return err
-	}
-
-	userName := "root"
-	if desc.ImageType == "system" && desc.OsType == "Windows" {
-		userName = "Administrator"
-	}
-
-	action, err := config.GetString("action")
-	if err != nil {
-		return err
-	}
-
-	ihost, err := host.GetIHost()
-	if err != nil {
-		return err
-	}
-
-	if action == "create" {
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-
-			iVM, createErr := ihost.CreateVM(&desc)
-			if createErr != nil {
-				return nil, createErr
-			}
-			guest.SetExternalId(task.GetUserCred(), iVM.GetGlobalId())
-
-			log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId())
-			err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
-			if err != nil {
-				return nil, err
-			}
-			log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId())
-
-			iVM, err = ihost.GetIVMById(iVM.GetGlobalId())
-			if err != nil {
-				log.Errorf("cannot find vm %s", err)
-				return nil, err
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, userName, desc.Password, action)
-
-			return data, nil
-		})
-	} else if action == "deploy" {
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-
-		params := task.GetParams()
-		log.Debugf("Deploy VM params %s", params.String())
-
-		deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false)
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-
-			if len(desc.UserData) > 0 {
-				err := iVM.UpdateUserData(desc.UserData)
-				if err != nil {
-					log.Errorf("update userdata fail %s", err)
-				}
-			}
-
-			err := iVM.DeployVM(ctx, desc.Name, desc.Password, desc.PublicKey, deleteKeypair, desc.Description)
-			if err != nil {
-				return nil, err
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, userName, desc.Password, action)
-
-			return data, nil
-		})
-	} else if action == "rebuild" {
-
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			if len(desc.UserData) > 0 {
-				err := iVM.UpdateUserData(desc.UserData)
-				if err != nil {
-					log.Errorf("update userdata fail %s", err)
-				}
-			}
-
-			diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB)
-			if err != nil {
-				return nil, err
-			}
-
-			log.Debugf("VMrebuildRoot %s new diskID %s, wait status ready ...", iVM.GetGlobalId(), diskId)
-
-			err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
-			if err != nil {
-				return nil, err
-			}
-			log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId())
-
-			maxWaitSecs := 300
-			waited := 0
-
-			for {
-				// hack, wait disk number consistent
-				idisks, err := iVM.GetIDisks()
-				if err != nil {
-					log.Errorf("fail to find VM idisks %s", err)
-					return nil, err
-				}
-				if len(idisks) < len(desc.DataDisks)+1 || idisks[0].GetGlobalId() != diskId {
-					if waited > maxWaitSecs {
-						log.Errorf("inconsistent disk number, wait timeout, must be something wrong on remote")
-						return nil, cloudprovider.ErrTimeout
-					}
-					if len(idisks) < len(desc.DataDisks)+1 {
-						log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1)
-					}
-					if len(idisks) > 0 && idisks[0].GetGlobalId() != diskId {
-						log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId)
-					}
-					time.Sleep(time.Second * 5)
-					waited += 5
-				} else {
-					break
-				}
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, userName, desc.Password, action)
-
-			return data, nil
-		})
-
-	} else {
-		log.Errorf("RequestDeployGuestOnHost: Action %s not supported", action)
-		return fmt.Errorf("Action %s not supported", action)
-	}
-
-	return nil
-} */
-
 func (self *SAliyunGuestDriver) AllowReconfigGuest() bool {
 	return true
 }

pkg/compute/guestdrivers/aws.go

@@ -67,6 +67,10 @@ func fetchAwsUserName(desc cloudprovider.SManagedVMCreateConfig) string {
 	}
 }
 
+func (self *SAwsGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovider.SManagedVMCreateConfig) bool {
+	return true
+}
+
 func (self *SAwsGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
 	// return fetchAwsUserName(desc)
 	return api.VM_AWS_DEFAULT_LOGIN_USER
@@ -76,6 +80,10 @@ func (self *SAwsGuestDriver) GetHypervisor() string {
 	return api.HYPERVISOR_AWS
 }
 
+func (self *SAwsGuestDriver) GetProvider() string {
+	return api.CLOUD_PROVIDER_AWS
+}
+
 func (self *SAwsGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_GP2_SSD
 }
@@ -148,137 +156,6 @@ func (self *SAwsGuestDriver) GetGuestInitialStateAfterRebuild() string {
 	return api.VM_READY
 }
 
-/*func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
-	config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams())
-	if err != nil {
-		log.Errorf("GetDeployConfigOnHost error: %v", err)
-		return err
-	}
-	log.Debugf("RequestDeployGuestOnHost: %s", config)
-	desc := cloudprovider.SManagedVMCreateConfig{}
-	if err := desc.GetConfig(config); err != nil {
-		return err
-	}
-	action, err := config.GetString("action")
-	if err != nil {
-		return err
-	}
-
-	ihost, err := host.GetIHost()
-	if err != nil {
-		return err
-	}
-
-	username := fetchAwsUserName(desc)
-
-	switch action {
-	case "create":
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-
-			iVM, createErr := ihost.CreateVM(&desc)
-			if createErr != nil {
-				return nil, createErr
-			}
-
-			guest.SetExternalId(task.GetUserCred(), iVM.GetGlobalId())
-
-			log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId())
-			err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800)
-			if err != nil {
-				return nil, err
-			}
-			log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId())
-
-			iVM, err = ihost.GetIVMById(iVM.GetGlobalId())
-			if err != nil {
-				log.Errorf("cannot find vm %s", err)
-				return nil, err
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, username, desc.Password, action)
-			return data, nil
-		})
-	case "deploy":
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-
-		params := task.GetParams()
-		log.Debugf("Deploy VM params %s", params.String())
-		deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false)
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			err := iVM.DeployVM(ctx, desc.Name, desc.Password, desc.PublicKey, deleteKeypair, desc.Description)
-			if err != nil {
-				return nil, err
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, username, desc.Password, action)
-			return data, nil
-		})
-	case "rebuild":
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB)
-			if err != nil {
-				return nil, err
-			}
-
-			log.Debugf("VMrebuildRoot %s new diskID %s, wait status ready ...", iVM.GetGlobalId(), diskId)
-
-			err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800)
-			if err != nil {
-				return nil, err
-			}
-			log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId())
-
-			maxWaitSecs := 300
-			waited := 0
-
-			for {
-				// hack, wait disk number consistent
-				idisks, err := iVM.GetIDisks()
-				if err != nil {
-					log.Errorf("fail to find VM idisks %s", err)
-					return nil, err
-				}
-				if len(idisks) < len(desc.DataDisks)+1 {
-					if waited > maxWaitSecs {
-						log.Errorf("inconsistent disk number, wait timeout, must be something wrong on remote")
-						return nil, cloudprovider.ErrTimeout
-					}
-					log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1)
-					time.Sleep(time.Second * 5)
-					waited += 5
-				} else {
-					if idisks[0].GetGlobalId() != diskId {
-						log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId)
-						return nil, fmt.Errorf("inconsistent sys disk id after rebuild root")
-					}
-
-					break
-				}
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, username, desc.Password, action)
-
-			return data, nil
-		})
-	default:
-		log.Errorf("RequestDeployGuestOnHost: Action %s not supported", action)
-		return fmt.Errorf("Action %s not supported", action)
-	}
-
-	return nil
-}*/
-
 func (self *SAwsGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool {
 	return false
 }

pkg/compute/guestdrivers/azure.go

@@ -20,18 +20,15 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cmdline"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/util/ansible"
 	"yunion.io/x/onecloud/pkg/util/billing"
 )
 
@@ -48,6 +45,10 @@ func (self *SAzureGuestDriver) GetHypervisor() string {
 	return api.HYPERVISOR_AZURE
 }
 
+func (self *SAzureGuestDriver) GetProvider() string {
+	return api.CLOUD_PROVIDER_AZURE
+}
+
 func (self *SAzureGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_STANDARD_LRS
 }
@@ -125,10 +126,6 @@ func (self *SAzureGuestDriver) ValidateUpdateData(ctx context.Context, userCred
 	if err != nil {
 		return nil, err
 	}
-	input, err = self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input)
-	if err != nil {
-		return nil, err
-	}
 	if input.Name != "" {
 		return nil, httperrors.NewInputParameterError("cannot support change azure instance name")
 	}
@@ -144,166 +141,15 @@ func (self *SAzureGuestDriver) GetGuestInitialStateAfterRebuild() string {
 }
 
 func (self *SAzureGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string {
-	return ansible.PUBLIC_CLOUD_ANSIBLE_USER
+	return api.VM_AZURE_DEFAULT_LOGIN_USER
 }
 
-/* func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
-	config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams())
-	if err != nil {
-		log.Errorf("GetDeployConfigOnHost error: %v", err)
-		return err
+func (self *SAzureGuestDriver) GetGuestSecgroupVpcid(guest *models.SGuest) (string, error) {
+	host := guest.GetHost()
+	if host != nil && strings.HasSuffix(host.Name, "-classic") {
+		return "classic", nil
 	}
-	desc := cloudprovider.SManagedVMCreateConfig{}
-	if err := desc.GetConfig(config); err != nil {
-		return err
-	}
-
-	action, err := config.GetString("action")
-	if err != nil {
-		return err
-	}
-	ihost, err := host.GetIHost()
-	if err != nil {
-		return err
-	}
-	if action == "create" {
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			if len(desc.Password) == 0 {
-				//Azure创建必须要设置密码
-				desc.Password = seclib2.RandomPassword2(12)
-			}
-
-			iVM, createErr := ihost.CreateVM(&desc)
-			if createErr != nil {
-				return nil, createErr
-			}
-
-			guest.SetExternalId(task.GetUserCred(), iVM.GetGlobalId())
-
-			log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId())
-			if err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil {
-				return nil, err
-			}
-			if iVM, err = ihost.GetIVMById(iVM.GetGlobalId()); err != nil {
-				log.Errorf("cannot find vm %s", err)
-				return nil, err
-			}
-
-			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, desc.Password, action)
-			return data, nil
-		})
-	} else if action == "deploy" {
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-		params := task.GetParams()
-		log.Debugf("Deploy VM params %s", params.String())
-		deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false)
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			err := iVM.DeployVM(ctx, desc.Name, desc.Password, desc.PublicKey, deleteKeypair, desc.Description)
-			if err != nil {
-				return nil, err
-			}
-			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, desc.Password, action)
-			return data, nil
-		})
-	} else if action == "rebuild" {
-		iVM, err := ihost.GetIVMById(guest.GetExternalId())
-		if err != nil || iVM == nil {
-			log.Errorf("cannot find vm %s", err)
-			return fmt.Errorf("cannot find vm")
-		}
-
-		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-			_, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB)
-			if err != nil {
-				return nil, err
-			}
-
-			log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId())
-			data := fetchIVMinfo(desc, iVM, guest.Id, ansible.PUBLIC_CLOUD_ANSIBLE_USER, desc.Password, action)
-
-			return data, nil
-		})
-
-	} else {
-		return fmt.Errorf("Action %s not supported", action)
-	}
-	return nil
-} */
-
-func (self *SAzureGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error {
-	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-		ihost, err := host.GetIHost()
-		if err != nil {
-			return nil, err
-		}
-		iVM, err := ihost.GetIVMById(guest.ExternalId)
-		if err != nil {
-			return nil, err
-		}
-
-		if fwOnly, _ := task.GetParams().Bool("fw_only"); fwOnly {
-			vpcID := "normal"
-			if strings.HasSuffix(host.Name, "-classic") {
-				vpcID = "classic"
-			}
-			iregion, err := host.GetIRegion()
-			if err != nil {
-				return nil, err
-			}
-
-			lockman.LockRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s", guest.SecgrpId, vpcID))
-			defer lockman.ReleaseRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s", guest.SecgrpId, vpcID))
-
-			secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), guest.SecgrpId, vpcID, host.GetRegion().Id, host.ManagerId)
-			if secgroupCache == nil {
-				return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s vpc: %s", guest.SecgrpId, vpcID)
-			}
-			extID, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpcID, guest.GetSecgroupName(), "", guest.GetSecRules())
-			if err != nil {
-				return nil, err
-			}
-			if err = secgroupCache.SetExternalId(task.GetUserCred(), extID); err != nil {
-				return nil, err
-			}
-			return nil, iVM.AssignSecurityGroup(extID)
-		}
-
-		iDisks, err := iVM.GetIDisks()
-		if err != nil {
-			return nil, err
-		}
-		disks := make([]models.SDisk, 0)
-		for _, guestdisk := range guest.GetDisks() {
-			disk := guestdisk.GetDisk()
-			disks = append(disks, *disk)
-		}
-
-		added := make([]models.SDisk, 0)
-		commondb := make([]models.SDisk, 0)
-		commonext := make([]cloudprovider.ICloudDisk, 0)
-		removed := make([]cloudprovider.ICloudDisk, 0)
-
-		if err := compare.CompareSets(disks, iDisks, &added, &commondb, &commonext, &removed); err != nil {
-			return nil, err
-		}
-		for _, disk := range removed {
-			if err := iVM.DetachDisk(ctx, disk.GetId()); err != nil {
-				return nil, err
-			}
-		}
-		for _, disk := range added {
-			if err := iVM.AttachDisk(ctx, disk.ExternalId); err != nil {
-				return nil, err
-			}
-		}
-		return nil, nil
-	})
-	return nil
+	return api.NORMAL_VPC_ID, nil
 }
 
 func (self *SAzureGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool {

pkg/compute/guestdrivers/baremetals.go

@@ -49,6 +49,10 @@ func (self *SBaremetalGuestDriver) GetHypervisor() string {
 	return api.HYPERVISOR_BAREMETAL
 }
 
+func (self *SBaremetalGuestDriver) GetProvider() string {
+	return api.CLOUD_PROVIDER_ONECLOUD
+}
+
 func (self *SBaremetalGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_LOCAL
 }
@@ -312,23 +316,11 @@ func (self *SBaremetalGuestDriver) StartGuestSyncstatusTask(guest *models.SGuest
 }
 
 func (self *SBaremetalGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) {
-	/*var confs = make([]api.BaremetalDiskConfig, 0)*/
-	//jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config")
-	//for i := 0; i < len(jsonArray); i += 1 {
-	//// for data.Contains(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) {
-	//desc, _ := jsonArray[i].GetString() // data.GetString(fmt.Sprintf("baremetal_disk_config.%d", len(confs)))
-	//data.Remove(fmt.Sprintf("baremetal_disk_config.%d", i))
-	//bmConf, err := baremetal.ParseDiskConfig(desc)
-	//if err != nil {
-	//return nil, httperrors.NewInputParameterError("baremetal_disk_config.%d", i)
-	//}
-	//confs = append(confs, bmConf)
-	//}
-	//if len(confs) == 0 {
-	//bmConf, _ := baremetal.ParseDiskConfig("")
-	//confs = append(confs, bmConf)
-	//}
-	/*data.Set("baremetal_disk_config", jsonutils.Marshal(confs))*/
+	if len(input.BaremetalDiskConfigs) != 0 {
+		if err := baremetal.ValidateDiskConfigs(input.BaremetalDiskConfigs); err != nil {
+			return nil, httperrors.NewInputParameterError("Invalid raid config: %v", err)
+		}
+	}
 	if len(input.Disks) <= 0 {
 		return nil, httperrors.NewInputParameterError("Root disk must be present")
 	}
@@ -377,11 +369,6 @@ func (self *SBaremetalGuestDriver) CheckDiskTemplateOnStorage(ctx context.Contex
 	return nil
 }
 
-func (self *SBaremetalGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
-	task.SetStage("OnDeployGuestSyncstatusComplete", nil)
-	return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId())
-}
-
 func (self *SBaremetalGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error {
 	if data.Contains("disks") {
 		disks, _ := data.GetArray("disks")