Merge pull request #14970 from swordqiu/automated-cherry-pick-of-#14967-upstream-release-3.7

Automated cherry pick of #14967: fix: set vfs_cache_pressure to 350

Makefile

@@ -268,7 +268,8 @@ dep:
 	@$(MAKE) mod
 
 mod:
-	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
+	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v sqlchemy))
+	go get -d yunion.io/x/sqlchemy@v1.0.0
 	go mod tidy
 	go mod vendor -v
 
@@ -330,7 +331,7 @@ image:
 .PHONY: image
 
 image-telegraf-raid-plugin:
-	VERSION=release-1.6.1 ARCH=all make image telegraf-raid-plugin
+	VERSION=release-1.6.3 ARCH=all make image telegraf-raid-plugin
 
 %:
 	@:

build/docker/Dockerfile.ansibleserver

@@ -1,7 +1,7 @@
 FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.5
 
 # install playbook and telegraf install pkg
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.3 /opt/yunion/playbook /opt/yunion/playbook
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.3 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/playbook /opt/yunion/playbook
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
 
 ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.baremetal-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.3
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.baremetal-base

@@ -1,6 +1,6 @@
 FROM --platform=linux/amd64 registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
-RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21080303.x86_64.rpm
-#RUN yum install -y http://192.168.122.112:8083/baremetal-pxerom-1.1.0-21080303.x86_64.rpm
+RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
+#RUN yum install -y http://192.168.23.50:8083/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 
 FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 

build/docker/Dockerfile.telegraf-raid-plugin

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.1
 
 ADD ./_output/centos-build/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/Makefile

@@ -23,7 +23,7 @@ climc-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/climc-base:$(CLIMC_BASE_VERSION) -f ./Dockerfile.climc-base .
 
-BAREMETAL_BASE_VERSION = v0.3.2
+BAREMETAL_BASE_VERSION = v0.3.3
 
 baremetal-base:
 	$(DOCKER_BUILDX)/baremetal-base:$(BAREMETAL_BASE_VERSION) -f ./Dockerfile.baremetal-base .

cmd/climc/shell/cloudproxy/proxy_agents.go

@@ -28,4 +28,5 @@ func init() {
 	cmd.List(&options.ProxyAgentListOptions{})
 	cmd.Show(&options.ProxyAgentShowOptions{})
 	cmd.Delete(&options.ProxyAgentDeleteOptions{})
+	cmd.Perform("purge-forwards", &options.ProxyAgentPurgeForwardsOptions{})
 }

cmd/climc/shell/compute/hosts.go

@@ -15,10 +15,12 @@
 package compute
 
 import (
+	"context"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -26,6 +28,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
@@ -523,9 +526,36 @@ func init() {
 			return e
 		}
 		i, e := modules.Hosts.GetLoginInfo(s, srvid, nil)
+		privateKey := ""
 		if e != nil {
-			return e
+			if httputils.ErrorCode(e) == 404 {
+				var err error
+				privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+				if err != nil {
+					return errors.Wrap(err, "fetch private key")
+				}
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewString(string(args.ID)), "ID")
+				ret, err := modules.Hosts.Get(s, args.ID, params)
+				if err != nil {
+					return errors.Wrap(err, "get host by ID")
+				}
+				ip, err := ret.GetString("access_ip")
+				if err != nil {
+					return errors.Wrap(err, "get the ip of the host")
+				}
+
+				jsonItem := jsonutils.NewDict()
+				jsonItem.Add(jsonutils.NewString(ip), "ip")
+				jsonItem.Add(jsonutils.NewString("root"), "username")
+				jsonItem.Add(jsonutils.NewString(""), "password")
+				jsonItem.Add(jsonutils.NewInt(22), "port")
+				i = jsonItem
+			} else {
+				return e
+			}
 		}
+
 		host, err := i.GetString("ip")
 		if err != nil {
 			return err
@@ -542,7 +572,7 @@ func init() {
 		if args.Port != 22 {
 			port = args.Port
 		}
-		sshCli, err := ssh.NewClient(host, port, user, passwd, "")
+		sshCli, err := ssh.NewClient(host, port, user, passwd, privateKey)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/compute/server_forward.go

@@ -47,7 +47,7 @@ func dump(input jsonutils.JSONObject) (*forwardInfo, error) {
 	}
 
 	if ret.ProxyPort <= 0 {
-		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyAddr)
+		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyPort)
 	}
 
 	return ret, nil

cmd/climc/shell/compute/servers.go

@@ -15,6 +15,7 @@
 package compute
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -89,6 +90,7 @@ func init() {
 	cmd.Perform("make-sshable", &options.ServerMakeSshableOptions{})
 	cmd.Perform("migrate-network", &options.ServerMigrateNetworkOptions{})
 	cmd.Perform("set-sshport", &options.ServerSetSshportOptions{})
+	cmd.Perform("have-agent", &options.ServerHaveAgentOptions{})
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))
@@ -99,6 +101,8 @@ func init() {
 	cmd.Get("make-sshable-cmd", new(options.ServerIdOptions))
 	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
 	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
+	cmd.Get("sshport", new(options.ServerIdOptions))
+
 	cmd.GetProperty(&options.ServerStatusStatisticsOptions{})
 
 	type ServerTaskShowOptions struct {
@@ -818,13 +822,15 @@ func init() {
 			return fmt.Errorf("Not found ip address from server %s", opts.ID)
 		}
 
+		privateKey := ""
 		params := jsonutils.NewDict()
 		if len(opts.Key) > 0 {
-			privateKey, e := ioutil.ReadFile(opts.Key)
+			key, e := ioutil.ReadFile(opts.Key)
 			if e != nil {
 				return e
 			}
-			params.Add(jsonutils.NewString(string(privateKey)), "private_key")
+			params.Add(jsonutils.NewString(string(key)), "private_key")
+			privateKey = string(key)
 		}
 
 		i, e := modules.Servers.GetLoginInfo(s, srvid, params)
@@ -832,14 +838,14 @@ func init() {
 			return e
 		}
 		passwd, err := i.GetString("password")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.Password != "" {
 			passwd = opts.Password
 		}
 		user, err := i.GetString("username")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.User != "" {
@@ -869,8 +875,22 @@ func init() {
 			}
 		}
 
+		if opts.UseCloudroot {
+			var err error
+			privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+			if err != nil {
+				return err
+			}
+			passwd = ""
+			user = "cloudroot"
+		}
+
 		var sshCli *ssh.Client
-		for sshCli, err = ssh.NewClient(host, port, user, passwd, ""); err != nil; {
+		err = nil
+		for ; sshCli == nil; sshCli, err = ssh.NewClient(host, port, user, passwd, privateKey) {
+			if err == nil {
+				continue
+			}
 			if opts.Host != "" {
 				return err
 			}
@@ -879,9 +899,9 @@ func init() {
 				return err
 			} else {
 				if vpcid != "default" {
-					forwardItem, err = openForward(s, srvid)
-					if err != nil {
-						return err
+					forwardItem, e = openForward(s, srvid)
+					if e != nil {
+						return e
 					}
 					host = forwardItem.ProxyAddr
 					port = forwardItem.ProxyPort

cmd/climc/shell/misc/licenses.go

@@ -15,6 +15,12 @@
 package misc
 
 import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -40,6 +46,40 @@ func init() {
 		return nil
 	})
 
+	type LicenseUploadOptions struct {
+		FILE string
+	}
+
+	R(&LicenseUploadOptions{}, "license-upload", "Upload license", func(s *mcclient.ClientSession, args *LicenseUploadOptions) error {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("license", args.FILE)
+		if err != nil {
+			return err
+		}
+		fh, err := os.Open(args.FILE)
+		if err != nil {
+			return err
+		}
+		defer fh.Close()
+		_, err = io.Copy(fileWriter, fh)
+		if err != nil {
+			return err
+		}
+		contentType := bodyWriter.FormDataContentType()
+		bodyWriter.Close()
+
+		header := http.Header{}
+		header.Set("Content-Type", contentType)
+		lic, err := modules.License.Upload(s, header, bodyBuf)
+		if err != nil {
+			return err
+		}
+
+		printObject(lic)
+		return nil
+	})
+
 	type LicenseShowOptions struct {
 		SERVICE string `help:"service name"  choices:"compute|service_tree"`
 	}

cmd/cloudmon/main.go

@@ -133,7 +133,7 @@ func main() {
 			for {
 				select {
 				case <-ticker.C:
-					log.Errorf("cmd: %s,provider: %v,end due to timeout: %s s", opts.SUBCOMMAND,
+					log.Errorf("cmd: %s,provider: %v,end due to timeout: %ds", opts.SUBCOMMAND,
 						suboptions.(*common.ReportOptions).Provider, suboptions.(*common.ReportOptions).Timeout)
 					os.Exit(3)
 				case <-endChan:

cmd/cryptool/shell/fernet.go

@@ -16,7 +16,6 @@ package shell
 
 import (
 	"fmt"
-	"time"
 
 	"yunion.io/x/pkg/util/timeutils"
 
@@ -110,7 +109,7 @@ func init() {
 		}
 		fmt.Println("primary key hash:", fm.PrimaryKeyHash())
 
-		ret := fm.Decrypt([]byte(args.MSG), time.Hour*-1)
+		ret := fm.Decrypt([]byte(args.MSG))
 		if len(ret) == 0 {
 			return fmt.Errorf("invalid message")
 		}

cmd/telegraf-raid-plugin/main.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 const (
@@ -128,6 +129,7 @@ func (c *RaidInfoCollector) CollectReportData() string {
 }
 
 func (c *RaidInfoCollector) Start() {
+	go procutils.WaitZombieLoop(context.TODO())
 	for {
 		c.runMain()
 		time.Sleep(time.Second * 1)

go.mod

@@ -55,7 +55,6 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.5.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
-	github.com/go-ini/ini v1.62.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
@@ -147,13 +146,13 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
-	yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349
+	yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
+	yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c
+	yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8
+	yunion.io/x/sqlchemy v1.0.0
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -235,8 +235,6 @@ github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
-github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
@@ -901,11 +899,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 h1:ESeezAb9LM2Dcy28DDrD2nI+h22qEf4MyAPXwuDO4uM=
-yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a h1:X/ucfLNcYsCfc4GFEX7R7GNWiDPr3vopLdnE4PuU46U=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o=
@@ -915,11 +913,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c h1:/ejlt78/0xod4bdkEThUuYsQ7vLajd48wpL2GMMSaXs=
-yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8 h1:+kywPlOttXAZvbxsVqR73B5GbSVgG3Xdo4socLWRkcg=
-yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v1.0.0 h1:fRtWxe1xxMR+NDVKYNuI89m2HGSl6aad2DZ+Bn/mZ4Q=
+yunion.io/x/sqlchemy v1.0.0/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/models/ansibleplaybooks.go

@@ -121,7 +121,7 @@ func (man *SAnsiblePlaybookManager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/ansibleserver/models/ansibleplaybooks_v2.go

@@ -43,7 +43,7 @@ type SAnsiblePlaybookV2 struct {
 
 	Playbook     string    `length:"text" nullable:"false" create:"required" get:"user"`
 	Inventory    string    `length:"text" nullable:"false" create:"required" get:"user"`
-	Requirements string    `length:"test" nullable:"false" create:"optional" get:"user"`
+	Requirements string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Files        string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Output       string    `length:"medium" get:"user"`
 	StartTime    time.Time `list:"user"`
@@ -118,7 +118,7 @@ func (man *SAnsiblePlaybookV2Manager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/apigateway/handler/auth.go

@@ -185,7 +185,22 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 
 	resp.Add(jsonutils.NewArray(retIdps...), "idps")
 
-	resp.Add(jsonutils.NewString(options.Options.ApiServer), "api_server")
+	// in case api_server changed but not synchronized from Keystone
+	// fetch this option directly from Keystone
+	var apiSrv string
+	commonCfg, err := modules.ServicesV3.GetSpecific(s, "common", "config", nil)
+	if err != nil {
+		log.Errorf("fetch common config error: %s", err)
+	} else {
+		apiSrv, err = commonCfg.GetString("config", "default", "api_server")
+		if err != nil {
+			log.Errorf("fetch api_server fail: %s (cfg: %s)", err, commonCfg)
+		}
+	}
+	if len(apiSrv) == 0 {
+		apiSrv = options.Options.ApiServer
+	}
+	resp.Add(jsonutils.NewString(apiSrv), "api_server")
 
 	return resp, nil
 }
@@ -1183,16 +1198,27 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 		return
 	}
 
-	oldPwd, _ := body.GetString("password_old")
-	newPwd, _ := body.GetString("password_new")
-	confirmPwd, _ := body.GetString("password_confirm")
-	passcode, _ := body.GetString("passcode")
+	input := struct {
+		PasswordOld     string
+		PasswordNew     string
+		PasswordConfirm string
+		Passcode        string
+	}{}
+	body.Unmarshal(&input)
 
-	if newPwd != confirmPwd {
+	if input.PasswordNew != input.PasswordConfirm {
 		httperrors.InputParameterError(ctx, w, "new password mismatch")
 		return
 	}
 
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordOld); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordOld = string(decPasswd)
+	}
+
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordNew); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordNew = string(decPasswd)
+	}
+
 	// 1.验证原密码正确，且idp_driver为空
 	if isIdpUser(user) {
 		httperrors.ForbiddenError(ctx, w, "not support reset user password")
@@ -1200,7 +1226,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	}
 
 	cliIp := netutils2.GetHttpRequestIp(req)
-	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), oldPwd, t.GetDomainName(), "", "", cliIp)
+	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), input.PasswordOld, t.GetDomainName(), "", "", cliIp)
 	if err != nil {
 		switch httperr := err.(type) {
 		case *httputils.JSONClientError:
@@ -1216,7 +1242,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	// 2.如果已开启MFA，验证 随机密码正确
 	if isMfaEnabled(user) {
-		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), passcode)
+		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), input.Passcode)
 		if err != nil {
 			httperrors.InputParameterError(ctx, w, "invalid passcode")
 			return
@@ -1225,7 +1251,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 
 	// 3.重置密码，
 	params := jsonutils.NewDict()
-	params.Set("password", jsonutils.NewString(newPwd))
+	params.Set("password", jsonutils.NewString(input.PasswordNew))
 	_, err = modules.UsersV3.Patch(s, t.GetUserId(), params)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)

pkg/apigateway/handler/dispatcher.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"time"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
@@ -51,7 +52,17 @@ func (h SHandler) Bind(app *appsrv.Application) {
 	if h.MiddlewareFunc != nil {
 		f = h.MiddlewareFunc(f)
 	}
-	app.AddHandler(h.Method, h.Prefix, f)
+	segs := appsrv.SplitPath(h.Prefix)
+	hi := appsrv.NewHandlerInfo(h.Method, segs, f, nil, "", nil)
+	if h.Method == "GET" {
+		hi.SetProcessTimeoutCallback(func(hdr *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+			if len(r.URL.Query().Get("export_keys")) > 0 {
+				return time.Hour * 2
+			}
+			return -time.Second
+		})
+	}
+	app.AddHandler3(hi)
 }
 
 func NewMethodHandlerFactory(method string, mf appsrv.MiddlewareFunc, prefix string) func(handleFunc, ...string) SHandler {

pkg/apigateway/handler/oidc.go

@@ -100,6 +100,7 @@ func handleOIDCAuth(ctx context.Context, w http.ResponseWriter, req *http.Reques
 	qs.Set("code", jsonutils.NewString(code))
 	qs.Set("state", jsonutils.NewString(auth.State))
 	redirUrl := addQuery(auth.RedirectUri, qs)
+	appsrv.DisableClientCache(w)
 	appsrv.SendRedirect(w, redirUrl)
 }
 

pkg/apihelper/apihelper.go

@@ -118,7 +118,7 @@ func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess
 	if s != nil {
 		token := s.GetToken()
 		expires := token.GetExpires()
-		if time.Now().Add(time.Hour).After(expires) {
+		if time.Now().Add(time.Hour).Before(expires) {
 			return s
 		}
 	}

pkg/apihelper/reflect.go

@@ -103,6 +103,7 @@ func GetModels(opts *GetModelsOptions) error {
 
 	listOptions := options.BaseListOptions{
 		Admin:        options.Bool(true),
+		Scope:        "system",
 		Details:      options.Bool(opts.IncludeDetails),
 		ShowEmulated: options.Bool(opts.IncludeEmulated),
 		Filter: []string{

pkg/apis/compute/api.go

@@ -396,6 +396,10 @@ type ServerCreateInput struct {
 	// emulate: BIOS, UEFI
 	Bios string `json:"bios"`
 
+	// Machine类型
+	// emulate: pc, q35
+	Machine string `json:"machine"`
+
 	// 启动顺序
 	// c: cdrome
 	// d: disk

pkg/apis/compute/baremetal_const.go

@@ -29,12 +29,13 @@ const (
 
 	DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE
 
-	DISK_DRIVER_MEGARAID   = "MegaRaid"
-	DISK_DRIVER_LINUX      = "Linux"
-	DISK_DRIVER_HPSARAID   = "HPSARaid"
-	DISK_DRIVER_MPT2SAS    = "Mpt2SAS"
-	DISK_DRIVER_MARVELRAID = "MarvelRaid"
-	DISK_DRIVER_PCIE       = "PCIE"
+	DISK_DRIVER_MEGARAID    = "MegaRaid"
+	DISK_DRIVER_LINUX       = "Linux"
+	DISK_DRIVER_HPSARAID    = "HPSARaid"
+	DISK_DRIVER_MPT2SAS     = "Mpt2SAS"
+	DISK_DRIVER_MARVELRAID  = "MarvelRaid"
+	DISK_DRIVER_ADAPTECRAID = "AdaptecRaid"
+	DISK_DRIVER_PCIE        = "PCIE"
 
 	HDD_DISK_SPEC_TYPE = "HDD"
 	SSD_DISK_SPEC_TYPE = "SSD"
@@ -60,6 +61,7 @@ var (
 		DISK_DRIVER_HPSARAID,
 		DISK_DRIVER_MPT2SAS,
 		DISK_DRIVER_MARVELRAID,
+		DISK_DRIVER_ADAPTECRAID,
 	)
 
 	DISK_DRIVERS = sets.NewString(

pkg/apis/compute/dnszone.go

@@ -125,7 +125,7 @@ type DnsZoneDetails struct {
 	// 关联vpc数量
 	VpcCount int `json:"vpc_count"`
 	// Cache info
-	CloudCaches []DnsZoneCacheDetails `json:"cloud_caches"`
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type DnsZoneListInput struct {
@@ -141,7 +141,8 @@ type DnsZoneListInput struct {
 	ZoneType string `json:"zone_type"`
 
 	// Filter dns zone By vpc
-	VpcId string `json:"vpc_id"`
+	VpcId     string `json:"vpc_id"`
+	WithCache bool   `json:"with_cache"`
 }
 
 type DnsZoneSyncStatusInput struct {

pkg/apis/compute/guest_const.go

@@ -176,6 +176,11 @@ const (
 	CPU_MODE_HOST = "host"
 )
 
+const (
+	VM_MACHINE_TYPE_PC  = "pc"
+	VM_MACHINE_TYPE_Q35 = "q35"
+)
+
 var VM_RUNNING_STATUS = []string{VM_START_START, VM_STARTING, VM_RUNNING, VM_BLOCK_STREAM, VM_BLOCK_STREAM_FAIL}
 var VM_CREATING_STATUS = []string{VM_CREATE_NETWORK, VM_CREATE_DISK, VM_START_DEPLOY, VM_DEPLOYING}
 
@@ -293,3 +298,7 @@ func Hypervisors2HostTypes(hypervisors []string) []string {
 	}
 	return hostTypes
 }
+
+// windows allow a maximal length of 15
+// http://support.microsoft.com/kb/909264
+const MAX_WINDOWS_COMPUTER_NAME_LENGTH = 15

pkg/apis/compute/guest_sshable.go

@@ -44,6 +44,13 @@ type GuestSshableOutput struct {
 	MethodTried []GuestSshableMethodData
 }
 
+type GuestHaveAgentInput struct {
+}
+
+type GuestHaveAgentOutput struct {
+	Have bool
+}
+
 type GuestMakeSshableInput struct {
 	User       string
 	PrivateKey string
@@ -59,6 +66,10 @@ type GuestMakeSshableCmdOutput struct {
 	ShellCmd string
 }
 
-type GuestSetSshPortInput struct {
+type GuestSetSshportInput struct {
+	Port int
+}
+
+type GuestSshportOutput struct {
 	Port int
 }

pkg/apis/compute/hoststorage.go

@@ -16,6 +16,8 @@ package compute
 
 import (
 	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
 )
 
 type HoststorageDetails struct {
@@ -58,3 +60,10 @@ type HoststorageListInput struct {
 
 	StorageFilterListInput
 }
+
+type HostStorageCreateInput struct {
+	apis.JoinResourceBaseCreateInput
+	StorageId  string `json:"storage_id"`
+	HostId     string `json:"host_id"`
+	MountPoint string `json:"mount_point"`
+}

pkg/apis/compute/network.go

@@ -209,22 +209,11 @@ type NetworkCreateInput struct {
 	BgpType string `json:"bgp_type"`
 }
 
-type NetworkDetails struct {
-	apis.SharableVirtualResourceDetails
-	WireResourceInfo
-
-	SNetwork
-
-	// 是否是内网
-	Exit bool `json:"exit"`
-	// 端口数量
-	Ports int `json:"ports"`
-	// 已使用端口数量
-	PortsUsed int `json:"ports_used"`
-	// 网卡数量
+type SNetworkNics struct {
+	// 虚拟机网卡数量
 	Vnics int `json:"vnics"`
 	// 裸金属网卡数量
-	BmVnics int `json:"bm_nics"`
+	BmVnics int `json:"bm_vnics"`
 	// 负载均衡网卡数量
 	LbVnics int `json:"lb_vnics"`
 	// 浮动Ip网卡数量
@@ -233,6 +222,48 @@ type NetworkDetails struct {
 	// 预留IP数量
 	ReserveVnics int `json:"reserve_vnics"`
 
+	// RDS网卡数量
+	RdsVnics int `json:"rds_vnics"`
+	// NAT网关网卡数量
+	NatVnics      int `json:"nat_vnics"`
+	BmReusedVnics int `json:"bm_reused_vnics"`
+
+	// 弹性网卡数量
+	NetworkinterfaceVnics int `json:"networkinterface_vnics"`
+
+	// 已使用端口数量
+	PortsUsed int `json:"ports_used"`
+
+	Total int `json:"total"`
+}
+
+func (self *SNetworkNics) SumTotal() {
+	self.Total = self.Vnics +
+		self.BmVnics +
+		self.LbVnics +
+		self.LbVnics +
+		self.EipVnics +
+		self.GroupVnics +
+		self.ReserveVnics +
+		self.RdsVnics +
+		self.NetworkinterfaceVnics +
+		self.NatVnics -
+		self.BmReusedVnics
+	self.PortsUsed = self.Total
+}
+
+type NetworkDetails struct {
+	apis.SharableVirtualResourceDetails
+	WireResourceInfo
+
+	SNetwork
+	SNetworkNics
+
+	// 是否是内网
+	Exit bool `json:"exit"`
+	// 端口数量
+	Ports int `json:"ports"`
+
 	// 路由信息
 	Routes    [][]string                 `json:"routes"`
 	Schedtags []SchedtagShortDescDetails `json:"schedtags"`

pkg/apis/compute/routetable_routesets.go

@@ -70,4 +70,7 @@ type RouteTableRouteSetListInput struct {
 	apis.StatusStandaloneResourceListInput
 	apis.ExternalizedResourceBaseListInput
 	RouteTableFilterList
+	Type        string `json:"type"`
+	NextHopType string `json:"next_hop_type"`
+	NextHopId   string `json:"next_hop_id"`
 }

pkg/apis/compute/secgroup.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"fmt"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
@@ -181,6 +182,7 @@ type SecgroupListInput struct {
 	RegionalFilterListInput
 
 	ManagedResourceListInput
+	WithCache bool `json:"witch_cache"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -252,6 +254,8 @@ type SecgroupDetails struct {
 	InRules []SecgroupRuleDetails `json:"in_rules"`
 	// 出方向规则信息
 	OutRules []SecgroupRuleDetails `json:"out_rules"`
+
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/storage_const.go

@@ -68,6 +68,7 @@ const (
 	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
 	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
 	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
+	STORAGE_HUAWEI_ESSD  = "ESSD"  // 急速型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/const.go

@@ -74,10 +74,20 @@ const (
 	OS_ARCH_AARCH64 = "aarch64"
 )
 
-func IsARM(osArch string) bool {
-	return utils.IsInStringArray(osArch, []string{
+var (
+	ARCH_X86 = []string{
+		OS_ARCH_X86,
+		OS_ARCH_I386,
+		OS_ARCH_X86_32,
+		OS_ARCH_X86_64,
+	}
+	ARCH_ARM = []string{
 		OS_ARCH_ARM,
 		OS_ARCH_AARCH32,
 		OS_ARCH_AARCH64,
-	})
+	}
+)
+
+func IsARM(osArch string) bool {
+	return utils.IsInStringArray(osArch, ARCH_ARM)
 }

pkg/apis/image/consts.go

@@ -41,6 +41,9 @@ const (
 	LocalFilePrefix = "file://"
 	S3Prefix        = "s3://"
 
+	IMAGE_STORAGE_DRIVER_LOCAL = "local"
+	IMAGE_STORAGE_DRIVER_S3    = "s3"
+
 	// image properties
 	IMAGE_OS_ARCH             = "os_arch"
 	IMAGE_OS_DISTRO           = "os_distribution"

pkg/apis/image/guestimage.go

@@ -26,8 +26,6 @@ type GuestImageDetails struct {
 	apis.SharableVirtualResourceDetails
 	SGuestImage
 
-	ImageIds []string `json:"image_ids"`
-
 	//Status     string               `json:"status"`
 	Size       int64          `json:"size"`
 	MinRamMb   int32          `json:"min_ram_mb"`

pkg/apis/output.go

@@ -40,7 +40,7 @@ type ModelBaseDetails struct {
 	CanUpdate bool `json:"can_update"`
 
 	// 资源不能更新的原因
-	UpdateFailReason string `json:"update_fail_reason"`
+	UpdateFailReason httperrors.Error `json:"update_fail_reason"`
 }
 
 type ModelBaseShortDescDetail struct {

pkg/appsrv/handlerinfo.go

@@ -83,6 +83,10 @@ func (this *SHandlerInfo) GetTags() map[string]string {
 	return this.tags
 }
 
+func NewHandlerInfo(method string, path []string, handler func(context.Context, http.ResponseWriter, *http.Request), metadata map[string]interface{}, name string, tags map[string]string) *SHandlerInfo {
+	return newHandlerInfo(method, path, handler, metadata, name, tags)
+}
+
 func newHandlerInfo(method string, path []string, handler func(context.Context, http.ResponseWriter, *http.Request), metadata map[string]interface{}, name string, tags map[string]string) *SHandlerInfo {
 	hand := SHandlerInfo{method: method, path: path,
 		handler:  handler,

pkg/baremetal/manager.go

@@ -185,6 +185,7 @@ func (m *SBaremetalManager) CleanBaremetal(bmId string) {
 func (m *SBaremetalManager) updateBaremetal(session *mcclient.ClientSession, bmId string) (jsonutils.JSONObject, error) {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.JSONTrue, "is_baremetal")
+	params.Add(jsonutils.JSONTrue, "not_sync_config")
 	obj, err := modules.Hosts.Put(session, bmId, params)
 	if err != nil {
 		return nil, err
@@ -2448,9 +2449,7 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) error {
 		}
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
 		return err
 	}
@@ -2458,6 +2457,7 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) error {
 	for tried := 0; !tool.IsAllDisksReady() && tried < maxTries; tried++ {
 		time.Sleep(5 * time.Second)
 		tool.RetrieveDiskInfo()
+		log.Warningf("disktool not ready string: %s", tool.DebugString())
 	}
 
 	if !tool.IsAllDisksReady() {
@@ -2528,11 +2528,9 @@ func (s *SBaremetalServer) DoPartitionDisk(term *ssh.Client) ([]*disktool.Partit
 		return nil, errors.Wrapf(err, "CalculateLayout")
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
-		return nil, errors.Wrapf(err, "RetrieveDiskInfo")
+		return nil, errors.Wrapf(err, "NewSSHPartitionTool")
 	}
 
 	disks, _ := s.desc.GetArray("disks")
@@ -2605,9 +2603,7 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 		return nil, err
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
 		return nil, err
 	}
@@ -2663,7 +2659,13 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 
 func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {
 	disks, _ := s.desc.GetArray("disks")
+
+	// calculate root partitions count
 	rootPartsCnt := len(parts) - len(disks) + 1
+	if len(parts) < len(disks) {
+		// HACK: rebuild root disk
+		rootPartsCnt = len(parts)
+	}
 	rootParts := parts[0:rootPartsCnt]
 	dataParts := parts[rootPartsCnt:]
 	idx := 0

pkg/baremetal/options/options.go

@@ -36,6 +36,7 @@ type BaremetalOptions struct {
 	TftpMaxTimeoutRetries  int    `default:"50" help:"Maximal tftp timeout retries, default is 50"`
 	LengthyWorkerCount     int    `default:"8" help:"Parallel worker count for lengthy tasks"`
 	ShortWorkerCount       int    `default:"8" help:"Parallel worker count for short-lived tasks"`
+	TaskWorkerCount        int    `default:"32" help:"Parallel worker count for tasks"`
 
 	DefaultIpmiPassword       string `help:"Default IPMI passowrd"`
 	DefaultStrongIpmiPassword string `help:"Default strong IPMI passowrd"`

pkg/baremetal/tasks/worker.go

@@ -24,16 +24,16 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/baremetal/options"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 var baremetalTaskWorkerMan *appsrv.SWorkerManager
 
-func init() {
-	baremetalTaskWorkerMan = appsrv.NewWorkerManager("BaremetalTaskWorkerManager", 8, 1024, false)
-}
-
 func GetWorkManager() *appsrv.SWorkerManager {
+	if baremetalTaskWorkerMan == nil {
+		baremetalTaskWorkerMan = appsrv.NewWorkerManager("BaremetalTaskWorkerManager", options.Options.TaskWorkerCount, 1024, false)
+	}
 	return baremetalTaskWorkerMan
 }
 
@@ -45,7 +45,7 @@ func OnStop() {
 }
 
 func ExecuteTask(task ITask, args interface{}) {
-	baremetalTaskWorkerMan.Run(func() {
+	GetWorkManager().Run(func() {
 		executeTask(task, args)
 	}, nil, nil)
 }

pkg/baremetal/utils/detect_storages/detect_storages.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"time"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
@@ -91,7 +92,7 @@ func DetectStorageInfo(term raid.IExecTerm, wait bool) ([]*baremetal.BaremetalSt
 			break
 		}
 	}
-	log.Infof("RaidDiskInfo: %#v, NonRaidSCSIDiskInfo: %#v, PCIEDiskInfo: %#v", raidDiskInfo, nonRaidDiskInfo, pcieDiskInfo)
+	log.Infof("RaidDiskInfo: %s, NonRaidSCSIDiskInfo: %s, PCIEDiskInfo: %s", jsonutils.Marshal(raidDiskInfo), jsonutils.Marshal(nonRaidDiskInfo), jsonutils.Marshal(pcieDiskInfo))
 	if len(nonRaidDiskInfo) < len(lvDiskInfo) {
 		return nil, nil, nil, fmt.Errorf("Fail to retrieve disk info")
 	}

pkg/baremetal/utils/disktool/disktool.go

@@ -288,7 +288,7 @@ func (ps *DiskPartitions) String() string {
 func (ps *DiskPartitions) DebugString() string {
 	partitionsStr := []string{}
 	for _, p := range ps.partitions {
-		partitionsStr = append(partitionsStr, fmt.Sprintf("%#v", *p))
+		partitionsStr = append(partitionsStr, fmt.Sprintf("{%#v}", *p))
 	}
 	return fmt.Sprintf("driver: %s, dev: %s, sectors: %d, partitions: %#v", ps.driver, ps.dev, ps.sectors, partitionsStr)
 }
@@ -599,7 +599,7 @@ func (tool *PartitionTool) DebugString() string {
 		return ret
 	}
 	for driver, disks := range tool.diskTable {
-		s := fmt.Sprintf("%s: %v", driver, disksString(disks))
+		s := fmt.Sprintf("(%s: %v)", driver, disksString(disks))
 		ret = append(ret, s)
 	}
 	return strings.Join(ret, "\n")
@@ -753,7 +753,7 @@ type SSHPartitionTool struct {
 	term *ssh.Client
 }
 
-func NewSSHPartitionTool(term *ssh.Client) *SSHPartitionTool {
+func newSSHPartitionTool(term *ssh.Client) *SSHPartitionTool {
 	tool := &SSHPartitionTool{
 		term: term,
 	}
@@ -761,6 +761,15 @@ func NewSSHPartitionTool(term *ssh.Client) *SSHPartitionTool {
 	return tool
 }
 
+func NewSSHPartitionTool(term *ssh.Client, layouts []baremetal.Layout) (*SSHPartitionTool, error) {
+	tool := newSSHPartitionTool(term)
+	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
+	if err := tool.RetrieveDiskInfo(); err != nil {
+		return nil, errors.Wrapf(err, "RetrieveDiskInfo")
+	}
+	return tool, nil
+}
+
 func (tool *SSHPartitionTool) Run(cmds ...string) ([]string, error) {
 	return tool.term.Run(cmds...)
 }

pkg/baremetal/utils/raid/adaptec/adaptec.go

@@ -0,0 +1,655 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package adaptec
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/stringutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	"yunion.io/x/onecloud/pkg/compute/baremetal"
+	"yunion.io/x/onecloud/pkg/util/regutils2"
+)
+
+func init() {
+	raid.RegisterDriver(api.DISK_DRIVER_ADAPTECRAID, NewAdaptecRaid)
+}
+
+const (
+	ControllerModeRaidExposeRaw = "RAID (Expose RAW)"
+	ControllerModeRaidHideRaw   = "RAID (Hide RAW)"
+	ControllerModeMixed         = "Mixed"
+)
+
+type AdaptecRaid struct {
+	term     raid.IExecTerm
+	adapters []*AdaptecRaidAdaptor
+}
+
+func NewAdaptecRaid(term raid.IExecTerm) raid.IRaidDriver {
+	return &AdaptecRaid{
+		term:     term,
+		adapters: make([]*AdaptecRaidAdaptor, 0),
+	}
+}
+
+func GetCommand(args ...string) string {
+	bin := "/opt/adaptec/arcconf"
+	return raid.GetCommand(bin, args...)
+}
+
+func (r *AdaptecRaid) GetName() string {
+	return baremetal.DISK_DRIVER_ADAPTECRAID
+}
+
+func (r *AdaptecRaid) ParsePhyDevs() error {
+	cmd := GetCommand("LIST")
+	ret, err := r.term.Run(cmd)
+	if err != nil {
+		return errors.Wrap(err, "list controllers")
+	}
+	if err := r.parsePhyDevs(ret); err != nil {
+		return errors.Wrap(err, "parse physical device")
+	}
+	if len(r.adapters) == 0 {
+		return errors.Errorf("Not found adaptec raid controller")
+	}
+	return nil
+}
+
+func (r *AdaptecRaid) CleanRaid() error {
+	for _, ada := range r.adapters {
+		ada.removeJBODDisks()
+		ada.RemoveLogicVolumes()
+	}
+	return nil
+}
+
+func (r *AdaptecRaid) GetAdapters() []raid.IRaidAdapter {
+	ret := make([]raid.IRaidAdapter, 0)
+	for _, a := range r.adapters {
+		ret = append(ret, a)
+	}
+	return ret
+}
+
+func (r *AdaptecRaid) PreBuildRaid(confs []*api.BaremetalDiskConfig, adapterIdx int) error {
+	return nil
+}
+
+var (
+	adaptorIDPatter = regexp.MustCompile(`Controller (?P<idx>[0-9]+):`)
+)
+
+func getAdaptorIndex(line string) int {
+	adapStr := regutils2.GetParams(adaptorIDPatter, line)["idx"]
+	if adapStr == "" {
+		return -1
+	}
+	adapInt, err := strconv.Atoi(adapStr)
+	if err != nil {
+		log.Errorf("Parse adapator string %q id error: %v", line, err)
+		return -1
+	}
+	return adapInt
+}
+
+func (r *AdaptecRaid) parsePhyDevs(lines []string) error {
+	for _, line := range lines {
+		index := getAdaptorIndex(line)
+		if index == -1 {
+			continue
+		}
+		ada, err := NewAdaptecRaidAdaptor(index, r)
+		if err != nil {
+			return errors.Wrapf(err, "New raid adaptor %d", index)
+		}
+		r.adapters = append(r.adapters, ada)
+	}
+	return nil
+}
+
+type AdaptecRaidAdaptor struct {
+	*adaptorInfo
+
+	index int
+	raid  *AdaptecRaid
+	devs  []*AdaptecRaidPhyDev
+}
+
+var (
+	_ raid.IRaidAdapter = new(AdaptecRaidAdaptor)
+)
+
+func NewAdaptecRaidAdaptor(index int, raid *AdaptecRaid) (*AdaptecRaidAdaptor, error) {
+	adaptor := AdaptecRaidAdaptor{
+		index: index,
+		raid:  raid,
+	}
+	if err := adaptor.fillInfo(); err != nil {
+		return nil, errors.Wrapf(err, "fill adaptor %d info", adaptor.index)
+	}
+	if err := adaptor.fillDevices(); err != nil {
+		return nil, errors.Wrap(err, "fill physical devices")
+	}
+	/*
+	 * if !adaptor.isRaidHideRawMode() {
+	 * 	if err := adaptor.setControllerModeRaidHideRaw(); err != nil {
+	 * 		return nil, errors.Wrap(err, "set controller mode to raid hide raw")
+	 * 	}
+	 * }
+	 */
+	return &adaptor, nil
+}
+
+func (ada *AdaptecRaidAdaptor) GetIndex() int {
+	return ada.index
+}
+
+func (ada *AdaptecRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error {
+	if !ada.isRaidHideRawMode() {
+		if err := ada.setControllerModeRaidHideRaw(); err != nil {
+			return errors.Wrapf(err, "set controller mode to raid hide raw mode")
+		}
+	}
+	if err := ada.removeJBODDisks(); err != nil {
+		log.Warningf("remove all JBOD disks: %v", err)
+	}
+	// uninitialize all device to ready state
+	if err := ada.uninitializeAllDevice(); err != nil {
+		// return errors.Wrap(err, "uninitialize all devices")
+		log.Warningf("uninitializeAllDevice error: %v", err)
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) getInitializeCmd(args ...string) string {
+	newArgs := []string{"TASK", "START", fmt.Sprintf("%d", ada.GetIndex()), "DEVICE"}
+	newArgs = append(newArgs, args...)
+	newArgs = append(newArgs, "INITIALIZE", "noprompt")
+	return GetCommand(newArgs...)
+}
+
+func (ada *AdaptecRaidAdaptor) initializeDevice(channel string, id string) error {
+	cmd := ada.getInitializeCmd(channel, id)
+	_, err := ada.remoteRun(fmt.Sprintf("initialize device %s:%s", channel, id), cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) initializeAllDevice() error {
+	cmd := ada.getInitializeCmd("ALL")
+	_, err := ada.remoteRun("initialize all device", cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) uninitializeDevice(channel string, id string) error {
+	cmd := ada.getUninitializeCmd(channel, id)
+	_, err := ada.remoteRun(fmt.Sprintf("uninitialize device %s:%s", channel, id), cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) getUninitializeCmd(args ...string) string {
+	newArgs := []string{"TASK", "START", fmt.Sprintf("%d", ada.GetIndex()), "DEVICE"}
+	newArgs = append(newArgs, args...)
+	newArgs = append(newArgs, "UNINITIALIZE", "noprompt")
+	return GetCommand(newArgs...)
+}
+
+func (ada *AdaptecRaidAdaptor) uninitializeAllDevice() error {
+	cmd := ada.getUninitializeCmd("ALL")
+	_, err := ada.remoteRun("uninitialize all devices", cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) getTerm() raid.IExecTerm {
+	return ada.raid.term
+}
+
+func (ada *AdaptecRaidAdaptor) remoteRun(hint string, cmd string) ([]string, error) {
+	out, err := ada.getTerm().Run(cmd)
+	if err != nil {
+		return out, errors.Wrapf(err, "%q, out: %v", hint, out)
+	}
+	log.Debugf("remote run cmd %s %q successfully: %v", hint, cmd, out)
+	return out, nil
+}
+
+func (ada *AdaptecRaidAdaptor) fillInfo() error {
+	cmd := GetCommand("GETCONFIG", fmt.Sprintf("%d", ada.index), "AD")
+	ret, err := ada.remoteRun("get AD config", cmd)
+	if err != nil {
+		return err
+	}
+	info, err := getAdaptorInfo(ret)
+	if err != nil {
+		return errors.Wrap(err, "get adaptor info")
+	}
+	ada.adaptorInfo = info
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) fillDevices() error {
+	cmd := GetCommand("GETCONFIG", fmt.Sprintf("%d", ada.index), "PD")
+	ret, err := ada.remoteRun("get PD config", cmd)
+	if err != nil {
+		return err
+	}
+	devs := getPhyDevices(ada.GetIndex(), ret)
+	ada.devs = append(ada.devs, devs...)
+	return nil
+}
+
+func getPhyDevices(adapter int, lines []string) []*AdaptecRaidPhyDev {
+	dev := newPhyDev(adapter)
+	devs := make([]*AdaptecRaidPhyDev, 0)
+	for _, line := range lines {
+		if dev.parseLine(line) && dev.isComplete() {
+			tmpDev := dev
+			devs = append(devs, tmpDev)
+			dev = newPhyDev(adapter)
+		}
+	}
+	return devs
+}
+
+func (ada *AdaptecRaidAdaptor) getChannelId(storage *baremetal.BaremetalStorage) (string, string, error) {
+	if storage.Addr == "" {
+		return "", "", errors.Errorf("storage %#v addr is empty", storage)
+	}
+	parts := strings.Split(storage.Addr, ":")
+	if len(parts) != 2 {
+		return "", "", errors.Errorf("invalid storage %#v addr %q", storage, storage.Addr)
+	}
+	return parts[0], parts[1], nil
+}
+
+func (ada *AdaptecRaidAdaptor) getCreateCmd(args ...string) string {
+	newArgs := []string{"CREATE", fmt.Sprintf("%d", ada.GetIndex())}
+	newArgs = append(newArgs, args...)
+	return GetCommand(newArgs...)
+}
+
+// setControllerMode change adapter controller's mode
+//   Controller Modes  : 0  - RAID: Expose RAW
+//                     : 1  - Auto Volume Mode
+//                     : 2  - HBA Mode
+//                     : 3  - RAID: Hide RAW
+//                     : 4  - Simple Volume Mode
+//                     : 5  - Mixed
+func (ada *AdaptecRaidAdaptor) setControllerMode(mode int) error {
+	cmd := GetCommand("SETCONTROLLERMODE", fmt.Sprintf("%d", ada.GetIndex()), fmt.Sprintf("%d", mode), "noprompt")
+	if _, err := ada.remoteRun(fmt.Sprintf("set controller mode to %d", mode), cmd); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) setControllerModeRaidExposeRaw() error {
+	if err := ada.setControllerMode(0); err != nil {
+		return errors.Wrap(err, "mode raid expose raw")
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) setControllerModeRaidHideRaw() error {
+	if err := ada.setControllerMode(3); err != nil {
+		return errors.Wrap(err, "mode raid hide raw")
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) setControllerModeMixed() error {
+	if err := ada.setControllerMode(5); err != nil {
+		return errors.Wrap(err, "mode mixed")
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) buildJBOD(dev *baremetal.BaremetalStorage) error {
+	channel, id, err := ada.getChannelId(dev)
+	if err != nil {
+		return errors.Wrap(err, "get channel and id")
+	}
+	cmd := ada.getCreateCmd("JBOD", channel, id, "noprompt")
+	if out, err := ada.remoteRun("create JBOD", cmd); err != nil {
+		return errors.Wrapf(err, "run cmd %q, out: %q", cmd, out)
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) buildNonRaid(dev *baremetal.BaremetalStorage) error {
+	// set to raid expose raw mode or mixed mode
+	if !ada.isRaidExposeRawMode() && !ada.isMixedMode() {
+		err := func() error {
+			errs := []error{}
+			if err := ada.setControllerModeRaidExposeRaw(); err != nil {
+				errs = append(errs, err)
+			} else {
+				return nil
+			}
+			if err := ada.setControllerModeMixed(); err != nil {
+				errs = append(errs, err)
+			} else {
+				return nil
+			}
+			return errors.NewAggregate(errs)
+		}()
+		if err != nil {
+			return errors.Wrap(err, "set raid to expose raw or mixed mode")
+		}
+	}
+	// try build JBOD firstly
+	if err := ada.buildJBOD(dev); err != nil {
+		log.Warningf("try build JBOD error: %v", err)
+	} else {
+		return nil
+	}
+	// just uninitialize device when build JBOD fail
+	channel, id, err := ada.getChannelId(dev)
+	if err != nil {
+		return err
+	}
+	if err := ada.uninitializeDevice(channel, id); err != nil {
+		return errors.Wrapf(err, "uninitialize device %v when build jbod fail", dev)
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage) error {
+	for _, dev := range devs {
+		if err := ada.buildNonRaid(dev); err != nil {
+			return errors.Wrap(err, "build nonraid")
+		}
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) getBuildRaidCmd(level int, devs []*baremetal.BaremetalStorage) (string, error) {
+	if len(devs) == 0 {
+		return "", errors.Errorf("devices is empty")
+	}
+	args := []string{"LOGICALDRIVE", "MAX", fmt.Sprintf("%d", level)}
+	chIds := []string{}
+	for _, dev := range devs {
+		ch, id, err := ada.getChannelId(dev)
+		if err != nil {
+			return "", errors.Wrapf(err, "get device %v channel id", dev)
+		}
+		chIds = append(chIds, ch, id)
+	}
+	args = append(args, chIds...)
+	cmd := ada.getCreateCmd(args...)
+	return cmd, nil
+}
+
+func (ada *AdaptecRaidAdaptor) buildRaid(level int, devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
+	// initialize each devs
+	for _, dev := range devs {
+		channel, id, err := ada.getChannelId(dev)
+		if err != nil {
+			return errors.Wrapf(err, "get device %v channel id", dev)
+		}
+		if err := ada.initializeDevice(channel, id); err != nil {
+			// return errors.Wrapf(err, "initialize device")
+			log.Warningf("initialize device error: %v", err)
+		}
+	}
+
+	// TODO: support config to build raid params
+	cmd, err := ada.getBuildRaidCmd(level, devs)
+	if err != nil {
+		return err
+	}
+	_, err = ada.remoteRun(fmt.Sprintf("build raid %d", level), cmd)
+	return err
+}
+
+func (ada *AdaptecRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
+	return ada.buildRaid(0, devs, conf)
+}
+
+func (ada *AdaptecRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
+	return ada.buildRaid(1, devs, conf)
+}
+
+func (ada *AdaptecRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
+	return ada.buildRaid(5, devs, conf)
+}
+
+func (ada *AdaptecRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error {
+	return ada.buildRaid(10, devs, conf)
+}
+
+func (ada *AdaptecRaidAdaptor) GetDevices() []*baremetal.BaremetalStorage {
+	ret := []*baremetal.BaremetalStorage{}
+	for idx, dev := range ada.devs {
+		ret = append(ret, dev.ToBaremetalStorage(idx))
+	}
+	return ret
+}
+
+func (ada *AdaptecRaidAdaptor) GetLogicVolumes() ([]*raid.RaidLogicalVolume, error) {
+	cmd := GetCommand("GETCONFIG", fmt.Sprintf("%d", ada.index), "LD")
+	ret, err := ada.remoteRun("get logic volumes", cmd)
+	if err != nil {
+		return nil, fmt.Errorf("Get logic volumes error: %v", err)
+	}
+	return getLogicalVolumes(ada.index, ret)
+}
+
+func getLogicalVolumes(adapter int, lines []string) ([]*raid.RaidLogicalVolume, error) {
+	lvs := make([]*raid.RaidLogicalVolume, 0)
+	for _, line := range lines {
+		m := regutils2.SubGroupMatch(`Logical Device number\s+(?P<idx>\d+)`, line)
+		if len(m) > 0 {
+			idxStr := m["idx"]
+			idx, err := strconv.Atoi(idxStr)
+			if err != nil {
+				return nil, errors.Errorf("%s index str is not digit: %v", idxStr, err)
+			}
+			lvs = append(lvs, &raid.RaidLogicalVolume{
+				Index:   idx,
+				Adapter: adapter,
+			})
+		}
+	}
+	return lvs, nil
+}
+
+func (ada *AdaptecRaidAdaptor) removeJBODDisks() error {
+	cmd := GetCommand("DELETE", fmt.Sprintf("%d", ada.index), "JBOD", "ALL", "noprompt")
+	out, err := ada.remoteRun("delete JBOD disks", cmd)
+	if err != nil {
+		return errors.Wrapf(err, "delete all JBOD output: %q", out)
+	}
+	return nil
+}
+
+func (ada *AdaptecRaidAdaptor) RemoveLogicVolumes() error {
+	lvs, err := ada.GetLogicVolumes()
+	if err != nil {
+		return errors.Wrap(err, "get logic volumes")
+	}
+	if len(lvs) == 0 {
+		return nil
+	}
+	cmd := GetCommand("DELETE", fmt.Sprintf("%d", ada.index), "LOGICALDRIVE", "ALL", "noprompt")
+	out, err := ada.remoteRun("delete logical volumes", cmd)
+	if err != nil {
+		return errors.Wrapf(err, "delete all logicaldrive output: %q", out)
+	}
+	return nil
+}
+
+type adaptorInfo struct {
+	status string
+	mode   string
+	name   string
+	sn     string
+	wwn    string
+	slot   string
+}
+
+func (ada *adaptorInfo) key() string {
+	return ada.name + ada.sn
+}
+
+func (ada *adaptorInfo) isRaidExposeRawMode() bool {
+	return ada.mode == ControllerModeRaidExposeRaw
+}
+
+func (ada *adaptorInfo) isRaidHideRawMode() bool {
+	return ada.mode == ControllerModeRaidHideRaw
+}
+
+func (ada *adaptorInfo) isMixedMode() bool {
+	return ada.mode == ControllerModeMixed
+}
+
+func getAdaptorInfo(lines []string) (*adaptorInfo, error) {
+	ada := new(adaptorInfo)
+	for _, l := range lines {
+		key, val := stringutils.SplitKeyValue(l)
+		if len(key) == 0 {
+			continue
+		}
+		switch key {
+		case "Controller Status":
+			ada.status = val
+		case "Controller Mode":
+			ada.mode = val
+		case "Controller Model":
+			ada.name = val
+		case "Controller Serial Number":
+			ada.sn = val
+		case "Controller World Wide Name":
+			ada.wwn = val
+		case "Physical Slot":
+			ada.slot = val
+		}
+	}
+	if len(ada.key()) == 0 {
+		return nil, errors.Errorf("Not found SN and model name")
+	}
+	return ada, nil
+}
+
+type AdaptecRaidPhyDev struct {
+	*raid.RaidBasePhyDev
+	// channelId and deviceId is parsed by Reported Channel,Device(T:L)
+	// e.g.: Reported Channel,Device(T:L)       : 0,6(6:0)
+	channelId string
+	deviceId  string
+}
+
+func newPhyDev(adapter int) *AdaptecRaidPhyDev {
+	dev := &AdaptecRaidPhyDev{
+		RaidBasePhyDev: raid.NewRaidBasePhyDev(api.DISK_DRIVER_ADAPTECRAID),
+	}
+	dev.Adapter = adapter
+	return dev
+}
+
+func (dev *AdaptecRaidPhyDev) isComplete() bool {
+	if !dev.RaidBasePhyDev.IsComplete() {
+		return false
+	}
+	if dev.channelId == "" || dev.deviceId == "" {
+		return false
+	}
+	return true
+}
+
+func (dev *AdaptecRaidPhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalStorage {
+	s := dev.RaidBasePhyDev.ToBaremetalStorage(index)
+	s.Addr = fmt.Sprintf("%s:%s", dev.channelId, dev.deviceId)
+
+	return s
+}
+
+var (
+	channelDeviceRegexp = regexp.MustCompile(`Reported Channel,Device\(T:L\).*(?P<channel>\d+),(?P<device>\d+)\(\d+:\d+\)`)
+)
+
+func (dev *AdaptecRaidPhyDev) parseLine(line string) bool {
+	chanDevMatch := regutils2.GetParams(channelDeviceRegexp, line)
+	if len(chanDevMatch) != 0 {
+		channelId := chanDevMatch["channel"]
+		deviceId := chanDevMatch["device"]
+		dev.channelId = channelId
+		dev.deviceId = deviceId
+		return true
+	}
+	key, val := stringutils.SplitKeyValue(line)
+	if key == "" {
+		return false
+	}
+	switch key {
+	case "Total Size":
+		dat := strings.Split(val, " ")
+		szStr, unitStr := dat[0], dat[1]
+		var sz int64
+		szInt, err := strconv.Atoi(szStr)
+		if err != nil {
+			log.Errorf("Parse size string %s: %v", szStr, err)
+			return false
+		}
+		switch unitStr {
+		case "GB":
+			sz = int64(szInt * 1000 * 1000 * 100)
+		case "TB":
+			sz = int64(szInt * 1000 * 1000 * 1000 * 1000)
+		case "MB":
+			sz = int64(szInt * 1000 * 1000)
+		default:
+			log.Errorf("Unsupported unit: %s", unitStr)
+			return false
+		}
+		dev.Size = sz / 1024 / 1024
+	case "Model":
+		dev.Model = strings.Join(regexp.MustCompile(`\s+`).Split(val, -1), " ")
+	case "State":
+		dev.Status = val
+	case "SSD":
+		if val == "No" {
+			dev.Rotate = tristate.True
+		} else {
+			dev.Rotate = tristate.False
+		}
+	default:
+		return false
+	}
+	return true
+}

pkg/baremetal/utils/raid/adaptec/adaptec_test.go

@@ -0,0 +1,422 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package adaptec
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_getAdaptorIndex(t *testing.T) {
+	tests := []struct {
+		name string
+		args string
+		want int
+	}{
+		{
+			name: "Controllers found: 1",
+			args: "Controllers found: 1",
+			want: -1,
+		},
+		{
+			name: "   Controller 1:             : Optimal, Slot 4, RAID (Expose RAW), Adaptec ASR8805, 6A2263667CA, 50000D1701B47780",
+			args: "   Controller 1:             : Optimal, Slot 4, RAID (Expose RAW), Adaptec ASR8805, 6A2263667CA, 50000D1701B47780",
+			want: 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getAdaptorIndex(tt.args); got != tt.want {
+				t.Errorf("getAdaptorIndex() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getAdaptorInfo(t *testing.T) {
+	input := `Controllers found: 1
+----------------------------------------------------------------------
+Controller information
+----------------------------------------------------------------------
+   Controller Status                        : Optimal
+   Controller Mode                          : RAID (Expose RAW)
+   Channel description                      : SAS/SATA
+   Controller Model                         : Adaptec ASR8805
+   Controller Serial Number                 : 6A2263667CA
+   Controller World Wide Name               : 50000D1701B47780
+   Controller Alarm                         : Disabled
+   Physical Slot                            : 4
+   Temperature                              : 42 C/ 107 F (Normal)
+   Installed memory                         : 1024 MB
+   Host bus type                            : PCIe
+   Host bus speed                           : 8000 MHz
+   Host bus link width                      : 8 bit(s)/link(s)
+   Global task priority                     : High
+   Performance Mode                         : Default/Dynamic
+   PCI Device ID                            : 653
+   Stayawake period                         : Disabled
+   Spinup limit internal drives             : 0
+   Spinup limit external drives             : 0
+   Defunct disk drive count                 : 0
+   NCQ status                               : Enabled
+   Statistics data collection mode          : Disabled
+   Monitor Log Severity Level               : Informational
+   Global Max SAS Phy Link Rate             : 12 Gbps
+   Verify Write Setting                     : Not Applicable
+   Save Custom Defaults Setting             : Disabled
+   Smart Poll                               : Enabled
+   Error Tunable Profile                    : Normal
+   --------------------------------------------------------
+   Cache Properties
+   --------------------------------------------------------
+   Controller Cache Preservation            : Disabled
+   Global Physical Device Write Cache Policy: Drive Specific
+   --------------------------------------------------------
+   RAID Properties
+   --------------------------------------------------------
+   Logical devices/Failed/Degraded          : 0/0/0
+   Copyback                                 : Disabled
+   Automatic Failover                       : Enabled
+   Background consistency check             : Disabled
+   Background consistency check period      : 0
+   --------------------------------------------------------
+   Controller BIOS Setting Information
+   --------------------------------------------------------
+   Runtime BIOS                             : Enabled
+   Array BBS Support                        : Disabled
+   Physical Drives Displayed during POST    : Disabled
+   Backplane Mode                           : IBPI
+   BIOS Halt on Missing Drive Count         : 255
+   --------------------------------------------------------
+   Controller Version Information
+   --------------------------------------------------------
+   BIOS                                     : 7.18-0 (33556)
+   Firmware                                 : 7.18-0 (33556)
+   Driver                                   : 1.2-1 (50983)
+   Boot Flash                               : 7.18-0 (33556)
+   CPLD (Load version/ Flash version)       : 8/ 11
+   SEEPROM (Load version/ Flash version)    : 1/ 1
+   FCT Custom Init String Version           : 0x0
+
+   --------------------------------------------------------
+   Controller Cache Backup Unit Information
+   --------------------------------------------------------
+
+    Overall Backup Unit Status              : Not Present
+   --------------------------------------------------------
+   Connector information
+
+   --------------------------------------------------------
+   Connector #0
+      Connector Name                        : CN0
+
+      --------------------------------------
+   Lane Information
+      --------------------------------------
+      Lane #0
+         Channel ID                         : 0
+         Device ID                          : 0
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 3
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+      Lane #1
+         Channel ID                         : 0
+         Device ID                          : 1
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 2
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+      Lane #2
+         Channel ID                         : 0
+         Device ID                          : 2
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 0
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+      Lane #3
+         Channel ID                         : 0
+         Device ID                          : 3
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 1
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+
+   Connector #1
+      Connector Name                        : CN1
+
+      --------------------------------------
+   Lane Information
+      --------------------------------------
+      Lane #0
+         Channel ID                         : 0
+         Device ID                          : 4
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 5
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+      Lane #1
+         Channel ID                         : 0
+         Device ID                          : 5
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 6
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+
+      Lane #2
+         Channel ID                         : 0
+         Device ID                          : 6
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 4
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+            Attached PHY Identifier         : 0
+            Attached SAS Address            : 5000C50077893141
+            Negotiated Logical Link Rate    : PHY enabled - 6 Gbps
+
+      Lane #3
+         Channel ID                         : 0
+         Device ID                          : 7
+         SAS Address                        : 50000D1701B47780
+         PHY Identifier                     : 7
+         -----------------------------------
+         Lane SAS Phy Information
+         -----------------------------------
+            SAS Address                     : 50000D1701B47780
+            Attached PHY Identifier         : 0
+            Attached SAS Address            : 50000395B839D9AE
+            Negotiated Logical Link Rate    : PHY enabled - 6 Gbps
+
+
+
+
+Command completed successfully.`
+	lines := strings.Split(input, "\n")
+	info, err := getAdaptorInfo(lines)
+	if err != nil {
+		t.Errorf("getAdaptorInfo: %v", err)
+	}
+	assert := assert.New(t)
+	assert.Equal("Optimal", info.status)
+	assert.Equal("RAID (Expose RAW)", info.mode)
+	assert.Equal(true, info.isRaidExposeRawMode(), "should be raid expose raw mode")
+	assert.Equal("Adaptec ASR8805", info.name)
+	assert.Equal("6A2263667CA", info.sn)
+	assert.Equal("50000D1701B47780", info.wwn)
+	assert.Equal("4", info.slot)
+}
+
+func Test_getPhyDevices(t *testing.T) {
+	input := `Controllers found: 1
+----------------------------------------------------------------------
+Physical Device information
+----------------------------------------------------------------------
+      Device #0
+         Device is a Hard drive
+         State                              : Online
+         Block Size                         : 512 Bytes
+         Supported                          : Yes
+         Programmed Max Speed               : SAS 6.0 Gb/s
+         Transfer Speed                     : SAS 6.0 Gb/s
+         Reported Channel,Device(T:L)       : 0,6(6:0)
+         Reported Location                  : Connector 1, Device 2
+         Vendor                             : SEAGATE
+         Model                              : ST300MM0006
+         Firmware                           : LS0A
+         Serial number                      : S0K30A4N
+         World-wide name                    : 5000C50077893140
+         Reserved Size                      : 415982 KB
+         Used Size                          : 285696 MB
+         Unused Size                        : 64 KB
+         Total Size                         : 286102 MB
+         Write Cache                        : Disabled (write-through)
+         FRU                                : None
+         S.M.A.R.T.                         : No
+         S.M.A.R.T. warnings                : 0
+         Power State                        : Full rpm
+         Supported Power States             : Full rpm,Powered off
+         SSD                                : No
+         Temperature                        : 34 C/ 93 F
+      ----------------------------------------------------------------
+      Device Phy Information
+      ----------------------------------------------------------------
+         Phy #0
+            PHY Identifier                  : 0
+            SAS Address                     : 5000C50077893141
+            Attached PHY Identifier         : 4
+            Attached SAS Address            : 50000D1701B47780
+         Phy #1
+            PHY Identifier                  : 1
+            SAS Address                     : 5000C50077893142
+      ----------------------------------------------------------------
+      Runtime Error Counters
+      ----------------------------------------------------------------
+         Hardware Error Count               : 0
+         Medium Error Count                 : 0
+         Parity Error Count                 : 0
+         Link Failure Count                 : 0
+         Aborted Command Count              : 0
+         SMART Warning Count                : 0
+
+      Device #1
+         Device is a Hard drive
+         State                              : Online
+         Block Size                         : 512 Bytes
+         Supported                          : Yes
+         Programmed Max Speed               : SAS 6.0 Gb/s
+         Transfer Speed                     : SAS 6.0 Gb/s
+         Reported Channel,Device(T:L)       : 0,7(7:0)
+         Reported Location                  : Connector 1, Device 3
+         Vendor                             : TOSHIBA
+         Model                              : AL13SEB300
+         Firmware                           : DE0D
+         Serial number                      : 84T0A31SFRD6
+         World-wide name                    : 50000395B839D9AC
+         Reserved Size                      : 415982 KB
+         Used Size                          : 285696 MB
+         Unused Size                        : 64 KB
+         Total Size                         : 286102 MB
+         Write Cache                        : Disabled (write-through)
+         FRU                                : None
+         S.M.A.R.T.                         : No
+         S.M.A.R.T. warnings                : 0
+         Power State                        : Full rpm
+         Supported Power States             : Full rpm,Powered off
+         SSD                                : No
+         Temperature                        : 33 C/ 91 F
+      ----------------------------------------------------------------
+      Device Phy Information
+      ----------------------------------------------------------------
+         Phy #0
+            PHY Identifier                  : 0
+            SAS Address                     : 50000395B839D9AE
+            Attached PHY Identifier         : 7
+            Attached SAS Address            : 50000D1701B47780
+         Phy #1
+            PHY Identifier                  : 1
+            SAS Address                     : 50000395B839D9AF
+      ----------------------------------------------------------------
+      Runtime Error Counters
+      ----------------------------------------------------------------
+         Hardware Error Count               : 0
+         Medium Error Count                 : 0
+         Parity Error Count                 : 0
+         Link Failure Count                 : 0
+         Aborted Command Count              : 0
+         SMART Warning Count                : 0
+
+
+
+Command completed successfully.`
+	lines := strings.Split(input, "\n")
+	devs := getPhyDevices(1, lines)
+	assert := assert.New(t)
+	assert.Equal(2, len(devs), "physical devices should be 2")
+	dev1 := devs[0]
+	assert.Equal("0", dev1.channelId)
+	assert.Equal("6", dev1.deviceId)
+	assert.Equal("Online", dev1.Status)
+	assert.Equal("ST300MM0006", dev1.Model)
+	assert.Equal(true, dev1.Rotate.Bool())
+
+	dev2 := devs[1]
+	assert.Equal("0", dev2.channelId)
+	assert.Equal("7", dev2.deviceId)
+	assert.Equal("Online", dev2.Status)
+	assert.Equal("AL13SEB300", dev2.Model)
+	assert.Equal(true, dev2.Rotate.Bool())
+}
+
+func Test_getLogicalVolums(t *testing.T) {
+	input := `Controllers found: 1
+----------------------------------------------------------------------
+Logical device information
+----------------------------------------------------------------------
+Logical Device number 0
+   Logical Device name                      : LogicalDrv 0
+   Block Size of member drives              : 512 Bytes
+   RAID level                               : 1
+   Unique Identifier                        : 7D30DCD4
+   Status of Logical Device                 : Optimal
+   Additional details                       : Quick initialized
+   Size                                     : 285686 MB
+   Parity space                             : 285696 MB
+   Interface Type                           : Serial Attached SCSI
+   Device Type                              : HDD
+   Read-cache setting                       : Enabled
+   Read-cache status                        : On
+   Write-cache setting                      : Enabled
+   Write-cache status                       : On
+   Partitioned                              : No
+   Protected by Hot-Spare                   : No
+   Bootable                                 : Yes
+   Failed stripes                           : No
+   Power settings                           : Disabled
+   --------------------------------------------------------
+   Logical Device segment information
+   --------------------------------------------------------
+   Segment 0                                : Present (286102MB, SAS, HDD, Connector:1, Device:2)             S0K30A4N
+   Segment 1                                : Present (286102MB, SAS, HDD, Connector:1, Device:3)         84T0A31SFRD6
+
+
+
+Command completed successfully.`
+	lines := strings.Split(input, "\n")
+	lvs, err := getLogicalVolumes(1, lines)
+	if err != nil {
+		t.Errorf("getLogicalVolumes: %v", err)
+	}
+	assert := assert.New(t)
+	assert.Equal(1, len(lvs), "logical volume len should be 1")
+	lv := lvs[0]
+	assert.Equal(1, lv.Adapter)
+	assert.Equal(0, lv.Index)
+
+	inputNoLV := `Controllers found: 1
+----------------------------------------------------------------------
+Logical device information
+----------------------------------------------------------------------
+   No logical devices configured
+
+
+Command completed successfully.`
+	lines = strings.Split(inputNoLV, "\n")
+	lvs, err = getLogicalVolumes(1, lines)
+	if err != nil {
+		t.Errorf("getLogicalVolumes: %v", err)
+	}
+	assert.Equal(0, len(lvs), "logical volume len should be 0")
+}

pkg/baremetal/utils/raid/adaptec/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package adaptec // import "yunion.io/x/onecloud/pkg/baremetal/utils/raid/adaptec"

pkg/baremetal/utils/raid/drivers/drivers.go

@@ -24,6 +24,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
+	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/adaptec"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/hpssactl"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/megactl"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/mvcli"
@@ -64,6 +65,8 @@ func GetDriverByKernelModule(module string, term raid.IExecTerm) (raid.IRaidDriv
 		name = baremetal.DISK_DRIVER_HPSARAID
 	case raid.MODULE_MPT2SAS, raid.MODULE_MPT3SAS:
 		name = baremetal.DISK_DRIVER_MPT2SAS
+	case raid.MODULE_AACRAID, raid.MODULE_SMARTPQI:
+		name = baremetal.DISK_DRIVER_ADAPTECRAID
 	}
 	if name == "" {
 		return nil, errors.Errorf("Not support module %q", module)
@@ -93,13 +96,13 @@ func BuildRaid(driver raid.IRaidDriver, confs []*api.BaremetalDiskConfig, adapte
 	if adapter == nil {
 		return fmt.Errorf("Not found adapter by index %d", adapterIdx)
 	}
-	if err := buildRaid(adapter, confs); err != nil {
+	if err := buildRaid(driver, adapter, confs); err != nil {
 		return fmt.Errorf("Driver %s, adapter %d build raid: %v", driver.GetName(), adapterIdx, err)
 	}
 	return nil
 }
 
-func buildRaid(adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) error {
+func buildRaid(driver raid.IRaidDriver, adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) error {
 	if err := adapter.PreBuildRaid(confs); err != nil {
 		return fmt.Errorf("PreBuildRaid: %v", err)
 	}
@@ -139,6 +142,7 @@ func buildRaid(adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) erro
 		if err != nil {
 			return fmt.Errorf("Build raid %s: %v", conf.Conf, err)
 		}
+		log.Infof("Build %s:%d raid %s", driver.GetName(), adapter.GetIndex(), conf.Conf)
 	}
 	if len(nonDisks) > 0 {
 		if err := adapter.BuildNoneRaid(nonDisks); err != nil {

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -873,6 +873,10 @@ func (adapter *MegaRaidAdaptor) megacliBuildJBOD(devs []*baremetal.BaremetalStor
 	if !adapter.megacliIsJBODEnabled() {
 		return fmt.Errorf("JBOD not supported")
 	}
+	// try clear jbod disk of devices
+	if err := adapter.megacliClearJBODDisks(devs); err != nil {
+		log.Warningf("try clear megaraid jbod disks before make jbod: %s", err)
+	}
 	devIds := []string{}
 	for _, d := range devs {
 		devIds = append(devIds, GetSpecString(d))
@@ -914,10 +918,10 @@ func (adapter *MegaRaidAdaptor) storcliClearJBODDisks() error {
 	return errors.NewAggregate(errs)
 }
 
-func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
+func (adapter *MegaRaidAdaptor) megacliClearJBODDisks(devs []*baremetal.BaremetalStorage) error {
 	devIds := []string{}
-	for idx, dev := range adapter.devs {
-		devIds = append(devIds, GetSpecString(dev.ToBaremetalStorage(idx)))
+	for _, dev := range devs {
+		devIds = append(devIds, GetSpecString(dev))
 	}
 	errs := make([]error, 0)
 	for _, devId := range devIds {
@@ -930,9 +934,17 @@ func (adapter *MegaRaidAdaptor) megacliClearJBODDisks() error {
 	return errors.NewAggregate(errs)
 }
 
+func (adapter *MegaRaidAdaptor) megacliClearAllJBODDisks() error {
+	allDevs := make([]*baremetal.BaremetalStorage, 0)
+	for idx, dev := range adapter.devs {
+		allDevs = append(allDevs, dev.ToBaremetalStorage(idx))
+	}
+	return adapter.megacliClearJBODDisks(allDevs)
+}
+
 func (adapter *MegaRaidAdaptor) clearJBODDisks() {
-	if err := adapter.megacliClearJBODDisks(); err != nil {
-		log.Errorf("megacliClearJBODDisks error: %v", err)
+	if err := adapter.megacliClearAllJBODDisks(); err != nil {
+		log.Errorf("megacliClearAllJBODDisks error: %v", err)
 		log.Infof("try storcliClearJBODDisks")
 		if err := adapter.storcliClearJBODDisks(); err != nil {
 			log.Errorf("storcliClearJBODDisks error: %v", err)

pkg/baremetal/utils/raid/raid.go

@@ -36,6 +36,8 @@ const (
 	MODULE_HPSA     = "hpsa"
 	MODULE_MPT2SAS  = "mpt2sas"
 	MODULE_MPT3SAS  = "mpt3sas"
+	MODULE_AACRAID  = "aacraid"
+	MODULE_SMARTPQI = "smartpqi"
 )
 
 const (

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -38,7 +38,7 @@ type Mpt2SASRaidPhyDev struct {
 }
 
 func newMpt2SASRaidPhyDev(adapter int) *Mpt2SASRaidPhyDev {
-	b := raid.NewRaidBasePhyDev(baremetal.DISK_DRIVER_MARVELRAID)
+	b := raid.NewRaidBasePhyDev(baremetal.DISK_DRIVER_MPT2SAS)
 	b.Adapter = adapter
 	return &Mpt2SASRaidPhyDev{
 		RaidBasePhyDev: b,
@@ -116,6 +116,7 @@ func (dev *Mpt2SASRaidPhyDev) isComplete() bool {
 
 func (dev *Mpt2SASRaidPhyDev) ToBaremetalStorage(idx int) *baremetal.BaremetalStorage {
 	s := dev.RaidBasePhyDev.ToBaremetalStorage(idx)
+	s.Index = int64(idx)
 	s.Slot = dev.slot
 	s.Enclosure = dev.enclosure
 	s.Block = int64(dev.block)
@@ -296,7 +297,9 @@ func (adapter *Mpt2SASRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorag
 }
 
 func (adapter *Mpt2SASRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage) error {
-	return fmt.Errorf("Not impl")
+	// TODO: not impl
+	// return fmt.Errorf("Not impl")
+	return nil
 }
 
 func (adapter *Mpt2SASRaidAdaptor) RemoveLogicVolumes() error {
@@ -338,11 +341,25 @@ func (r *Mpt2SASRaid) ParsePhyDevs() error {
 	return r.parseAdapters(ret)
 }
 
+func getLineAdapterIndex(line string) int {
+	dat := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(line), -1)
+	if len(dat) == 0 {
+		return -1
+	}
+	if regexp.MustCompile(`\d+`).MatchString(dat[0]) {
+		if !regexp.MustCompile(`^\d+`).MatchString(dat[0]) {
+			return -1
+		}
+		idx, _ := strconv.Atoi(dat[0])
+		return idx
+	}
+	return -1
+}
+
 func (r *Mpt2SASRaid) parseAdapters(lines []string) error {
 	for _, line := range lines {
-		dat := regexp.MustCompile(`\s+`).Split(strings.TrimSpace(line), -1)
-		if regexp.MustCompile(`\d+`).MatchString(dat[0]) {
-			idx, _ := strconv.Atoi(dat[0])
+		idx := getLineAdapterIndex(line)
+		if idx >= 0 {
 			adapter := newMpt2SASRaidAdaptor(idx, r)
 			r.adapters = append(r.adapters, adapter)
 		}

pkg/baremetal/utils/raid/sas2iru/sas2iru_test.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sas2iru
+
+import "testing"
+
+func Test_getLineAdapterIndex(t *testing.T) {
+	tests := []struct {
+		line string
+		want int
+	}{
+		{
+			line: "0    SAS3008    100h    97h    00h:5eh:00h:00h    1000h    0097h",
+			want: 0,
+		},
+		{
+			line: "SAS3IRCU: Utility Completed Successfully.",
+			want: -1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.line, func(t *testing.T) {
+			if got := getLineAdapterIndex(tt.line); got != tt.want {
+				t.Errorf("getLineAdapterIndex() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/cloudcommon/db/caller.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -298,12 +299,40 @@ func FetchCustomizeColumns(
 	if ret[0].Len() != len(objs) {
 		return nil, httperrors.NewInternalServerError("Invalid FetchCustomizeColumns return value, inconsistent obj count: input %d != output %d", len(objs), ret[0].Len())
 	}
+
+	showReason := false
+	if query.Contains("show_fail_reason") {
+		showReason = true
+	}
+
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
 		// NOTE: don't use obj update jsonDict as retval
 		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
+		out := apis.ModelBaseDetails{
+			CanDelete: true,
+			CanUpdate: true,
+		}
+
+		err = ValidateDeleteCondition(objs[i].(IModel), ctx, jsonDict)
+		if err != nil {
+			out.CanDelete = false
+			if showReason {
+				out.DeleteFailReason = httperrors.NewErrorFromGeneralError(ctx, err)
+			}
+		}
+		err = ValidateUpdateCondition(objs[i].(IModel), ctx)
+		if err != nil {
+			out.CanUpdate = false
+			if showReason {
+				out.UpdateFailReason = httperrors.NewErrorFromGeneralError(ctx, err)
+			}
+		}
+		jsonDict.Update(jsonutils.Marshal(out))
+
 		retVal[i] = jsonDict
+
 	}
 	return retVal, nil
 }
@@ -333,3 +362,25 @@ func CustomizeDelete(model IModel, ctx context.Context, userCred mcclient.TokenC
 	}
 	return ValueToError(ret[0])
 }
+
+func ValidateDeleteCondition(model IModel, ctx context.Context, data jsonutils.JSONObject) error {
+	ret, err := call(model, "ValidateDeleteCondition", ctx, data)
+	if err != nil {
+		return httperrors.NewGeneralError(err)
+	}
+	if len(ret) != 1 {
+		return httperrors.NewInternalServerError("Invald ValidateDeleteCondition return value")
+	}
+	return ValueToError(ret[0])
+}
+
+func ValidateUpdateCondition(model IModel, ctx context.Context) error {
+	ret, err := call(model, "ValidateUpdateCondition", ctx)
+	if err != nil {
+		return httperrors.NewGeneralError(err)
+	}
+	if len(ret) != 1 {
+		return httperrors.NewInternalServerError("Invald ValidateUpdateCondition return value")
+	}
+	return ValueToError(ret[0])
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -30,7 +30,6 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
-	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
@@ -184,7 +183,7 @@ func applyListItemsSearchFilters(manager IModelManager, ctx context.Context, q *
 	return q, nil
 }
 
-func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery,
+func ApplyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery,
 	userCred mcclient.TokenCredential, filters []string, filterAny bool) (*sqlchemy.SQuery, error) {
 	conds := make([]sqlchemy.ICondition, 0)
 	schFields := searchFields(manager, userCred) // only filter searchable fields
@@ -298,7 +297,7 @@ func listItemQueryFiltersRaw(manager IModelManager,
 	filterAny, _ := query.Bool("filter_any")
 	filters := jsonutils.GetQueryStringArray(query, "filter")
 	if len(filters) > 0 {
-		q, err = applyListItemsGeneralFilters(manager, q, userCred, filters, filterAny)
+		q, err = ApplyListItemsGeneralFilters(manager, q, userCred, filters, filterAny)
 	}
 	jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter")
 	if len(jointFilter) > 0 {
@@ -708,12 +707,14 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 			// skip markerField in pagingConf
 			continue
 		}
-		colSpec := manager.TableSpec().ColumnSpec(orderByField)
-		if colSpec != nil {
-			if order == sqlchemy.SQL_ORDER_ASC {
-				q = q.Asc(orderByField)
-			} else {
-				q = q.Desc(orderByField)
+		for _, field := range q.QueryFields() {
+			if orderByField == field.Name() {
+				if order == sqlchemy.SQL_ORDER_ASC {
+					q = q.Asc(field)
+				} else {
+					q = q.Desc(field)
+				}
+				break
 			}
 		}
 	}
@@ -852,28 +853,6 @@ func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.J
 	return items, nil
 }
 
-func getModelExtraDetails(item IModel, ctx context.Context, showReason bool) apis.ModelBaseDetails {
-	out := apis.ModelBaseDetails{
-		CanDelete: true,
-		CanUpdate: true,
-	}
-	err := item.ValidateDeleteCondition(ctx)
-	if err != nil {
-		out.CanDelete = false
-		if showReason {
-			out.DeleteFailReason = httperrors.NewErrorFromGeneralError(ctx, err)
-		}
-	}
-	err = item.ValidateUpdateCondition(ctx)
-	if err != nil {
-		out.CanUpdate = false
-		if showReason {
-			out.UpdateFailReason = err.Error()
-		}
-	}
-	return out
-}
-
 func getModelItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, isHead bool) (jsonutils.JSONObject, error) {
 	appParams := appsrv.AppContextGetParams(ctx)
 	if appParams == nil && isHead {
@@ -899,7 +878,7 @@ func getModelItemDetails(manager IModelManager, item IModel, ctx context.Context
 }
 
 func GetItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
-	return getItemDetails(manager, item, ctx, userCred, nil)
+	return getItemDetails(manager, item, ctx, userCred, jsonutils.NewDict())
 }
 
 func getItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
@@ -1821,9 +1800,9 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 
 func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	// log.Debugf("deleteItem %s", jsonutils.Marshal(model))
-	err := model.ValidateDeleteCondition(ctx)
+
+	err := ValidateDeleteCondition(model, ctx, nil)
 	if err != nil {
-		log.Errorf("validate delete condition error: %s", err)
 		return nil, err
 	}
 

pkg/cloudcommon/db/db_joint_dispatcher.go

@@ -286,7 +286,7 @@ func (dispatcher *DBJointModelDispatcher) Detach(ctx context.Context, id1 string
 }
 
 func DetachJoint(ctx context.Context, userCred mcclient.TokenCredential, item IJointModel) error {
-	err := item.ValidateDeleteCondition(ctx)
+	err := ValidateDeleteCondition(item, ctx, nil)
 	if err != nil {
 		return err
 	}

pkg/cloudcommon/db/fetch.go

@@ -387,7 +387,8 @@ func FetchCheckQueryOwnerScope(ctx context.Context, userCred mcclient.TokenCrede
 		requireScope = queryScope
 	}
 	if doCheckRbac && requireScope.HigherThan(allowScope) {
-		return nil, scope, httperrors.NewForbiddenError("not enough privilege (require:%s,allow:%s,query:%s)",
+		return nil, scope, httperrors.NewForbiddenError("not enough privilege to do %s:%s:%s (require:%s,allow:%s,query:%s)",
+			consts.GetServiceType(), manager.KeywordPlural(), action,
 			requireScope, allowScope, queryScope)
 	}
 	return ownerId, queryScope, nil

pkg/cloudcommon/db/interface.go

@@ -183,7 +183,7 @@ type IModel interface {
 
 	// delete hooks
 	AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
-	ValidateDeleteCondition(ctx context.Context) error
+	// ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error
 	// CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 	PreDelete(ctx context.Context, userCred mcclient.TokenCredential)
 	MarkDelete() error

pkg/cloudcommon/db/metadataresource.go

@@ -204,12 +204,16 @@ func (meta *SMetadataResourceBaseModelManager) GetExportExtraKeys(keys stringuti
 }
 
 func (meta *SMetadataResourceBaseModelManager) ListItemExportKeys(manager IModelManager, q *sqlchemy.SQuery, keys stringutils2.SSortedStrings) *sqlchemy.SQuery {
+	keyMaps := map[string]bool{}
 	for _, key := range keys {
 		if strings.HasPrefix(key, TAG_EXPORT_KEY_PREFIX) {
 			tagKey := key[len(TAG_EXPORT_KEY_PREFIX):]
-			metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery()
-			q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id")))
-			q = q.AppendField(metaQ.Field("value", key))
+			if _, ok := keyMaps[strings.ToLower(tagKey)]; !ok {
+				metaQ := Metadata.Query("obj_id", "value").Equals("obj_type", manager.Keyword()).Equals("key", tagKey).SubQuery()
+				q = q.LeftJoin(metaQ, sqlchemy.Equals(q.Field("id"), metaQ.Field("obj_id")))
+				q = q.AppendField(metaQ.Field("value", key))
+				keyMaps[strings.ToLower(tagKey)] = true
+			}
 		}
 	}
 	return q

pkg/cloudcommon/db/modelbase.go

@@ -310,14 +310,7 @@ func (manager *SModelBaseManager) FetchCustomizeColumns(
 	fields stringutils2.SSortedStrings,
 	isList bool,
 ) []apis.ModelBaseDetails {
-	showReason := false
-	if query.Contains("show_fail_reason") {
-		showReason = true
-	}
 	ret := make([]apis.ModelBaseDetails, len(objs))
-	for i := range objs {
-		ret[i] = getModelExtraDetails(objs[i].(IModel), ctx, showReason)
-	}
 	return ret
 }
 
@@ -584,7 +577,7 @@ func (model *SModelBase) ValidateUpdateCondition(ctx context.Context) error {
 	return nil
 }
 
-func (model *SModelBase) ValidateDeleteCondition(ctx context.Context) error {
+func (model *SModelBase) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	return nil
 }
 

pkg/cloudcommon/db/opslog.go

@@ -51,7 +51,7 @@ type SOpsLog struct {
 	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
 	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`
 	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`
-	Notes   string `charset:"utf8" list:"user" create:"required"`
+	Notes   string `charset:"utf8" list:"user" create:"optional"`
 
 	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"optional" index:"true"`
 	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"optional"`
@@ -166,17 +166,29 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
 		ObjName: objName,
 		Action:  action,
 		Notes:   stringutils.Interface2String(notes),
-
-		ProjectId:       userCred.GetProjectId(),
-		Project:         userCred.GetProjectName(),
-		ProjectDomainId: userCred.GetProjectDomainId(),
-		ProjectDomain:   userCred.GetProjectDomain(),
-
-		UserId:   userCred.GetUserId(),
-		User:     userCred.GetUserName(),
-		DomainId: userCred.GetDomainId(),
-		Domain:   userCred.GetDomainName(),
-		Roles:    strings.Join(userCred.GetRoles(), ","),
+	}
+	if userCred == nil {
+		log.Warningf("Log event with empty userCred: objType=%s objId=%s objName=%s action=%s", model.Keyword(), objId, objName, action)
+		const unknown = "unknown"
+		opslog.ProjectId = unknown
+		opslog.Project = unknown
+		opslog.ProjectDomainId = unknown
+		opslog.ProjectDomain = unknown
+		opslog.UserId = unknown
+		opslog.User = unknown
+		opslog.DomainId = unknown
+		opslog.Domain = unknown
+		opslog.Roles = unknown
+	} else {
+		opslog.ProjectId = userCred.GetProjectId()
+		opslog.Project = userCred.GetProjectName()
+		opslog.ProjectDomainId = userCred.GetProjectDomainId()
+		opslog.ProjectDomain = userCred.GetProjectDomain()
+		opslog.UserId = userCred.GetUserId()
+		opslog.User = userCred.GetUserName()
+		opslog.DomainId = userCred.GetDomainId()
+		opslog.Domain = userCred.GetDomainName()
+		opslog.Roles = strings.Join(userCred.GetRoles(), ",")
 	}
 	opslog.SetModelManager(OpsLog, opslog)
 
@@ -363,7 +375,7 @@ func (self *SOpsLog) AllowDeleteItem(ctx context.Context, userCred mcclient.Toke
 	return false
 }
 
-func (self *SOpsLog) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SOpsLog) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	return httperrors.NewForbiddenError("not allow to delete log")
 }
 

pkg/cloudcommon/db/proxy/proxysetting.go

@@ -106,7 +106,7 @@ func (ps *SProxySetting) HttpTransportProxyFunc() httputils.TransportProxyFunc {
 	}
 }
 
-func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context) error {
+func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if ps.Id == proxyapi.ProxySettingId_DIRECT {
 		return httperrors.NewConflictError("DIRECT setting cannot be deleted")
 	}
@@ -122,7 +122,7 @@ func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context) error {
 				ps.Id, n, man.KeywordPlural())
 		}
 	}
-	return ps.SInfrasResourceBase.ValidateDeleteCondition(ctx)
+	return ps.SInfrasResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (ps *SProxySetting) AllowPerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {

pkg/cloudcommon/db/resourcebase.go

@@ -39,9 +39,9 @@ type SResourceBase struct {
 	// 资源被更新次数
 	UpdateVersion int `default:"0" nullable:"false" auto_version:"true" list:"user" json:"update_version"`
 	// 资源删除时间
-	DeletedAt time.Time `json:"deleted_at" list:"admin"`
+	DeletedAt time.Time `json:"deleted_at" list:"user"`
 	// 资源是否被删除
-	Deleted bool `nullable:"false" default:"false" json:"deleted" list:"admin"`
+	Deleted bool `nullable:"false" default:"false" json:"deleted" list:"user"`
 }
 
 type SResourceBaseManager struct {

pkg/cloudcommon/db/taskman/tasks.go

@@ -173,7 +173,7 @@ func (self *STask) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenC
 	return false
 }
 
-func (self *STask) ValidateDeleteCondition(ctx context.Context) error {
+func (self *STask) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	return httperrors.NewForbiddenError("forbidden")
 }
 

pkg/cloudcommon/options/changes.go

@@ -53,6 +53,9 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 	if oldOpts.EnableQuotaCheck != newOpts.EnableQuotaCheck {
 		consts.SetEnableQuotaCheck(newOpts.EnableQuotaCheck)
 	}
+	if oldOpts.ApiServer != newOpts.ApiServer {
+		log.Debugf("api_server changed from %s to %s", oldOpts.ApiServer, newOpts.ApiServer)
+	}
 	return changed
 }
 

pkg/cloudevent/models/cloudevents.go

@@ -297,47 +297,3 @@ func (manager *SCloudeventManager) GetPropertyDistinctField(ctx context.Context,
 	}
 	return jsonutils.Marshal(ret), nil
 }
-
-func (manager *SCloudeventManager) initDistinctField() error {
-	fileds, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword())
-	if err != nil {
-		return errors.Wrapf(err, "GetObjectDistinctFields")
-	}
-	if len(fileds) > 0 {
-		return nil
-	}
-	for _, key := range []string{"service", "resource_type", "action", "account", "manager", "provider", "brand"} {
-		values, err := db.FetchDistinctField(manager, key)
-		if err != nil {
-			return errors.Wrapf(err, "db.FetchDistinctField")
-		}
-		for _, value := range values {
-			if len(value) > 0 {
-				err = db.DistinctFieldManager.InsertOrUpdate(nil, manager, key, value)
-				if err != nil {
-					return errors.Wrapf(err, "DistinctFieldManager.InsertOrUpdate(%s, %s)", key, value)
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (manager *SCloudeventManager) InitializeData() error {
-	events := []SCloudevent{}
-	q := manager.Query().IsNullOrEmpty("brand")
-	err := db.FetchModelObjects(manager, q, &events)
-	if err != nil {
-		return err
-	}
-	for i := range events {
-		_, err = db.Update(&events[i], func() error {
-			events[i].Brand = events[i].Provider
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-	}
-	return manager.initDistinctField()
-}

pkg/cloudevent/models/cloudproviders.go

@@ -408,8 +408,7 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error)
 
 			ProxyFunc: proxyFunc,
 
-			SHCSOEndpoints:   delegate.Options.SHCSOEndpoints,
-			SApsaraEndpoints: delegate.Options.SApsaraEndpoints,
+			Options: jsonutils.Marshal(delegate.Options).(*jsonutils.JSONDict),
 		},
 	)
 }

pkg/cloudevent/models/initdb.go

@@ -33,7 +33,6 @@ func InitDB() error {
 		proxy.ProxySettingManager,
 
 		CloudproviderManager,
-		CloudeventManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/cloudid/models/cloudaccount.go

@@ -568,8 +568,7 @@ func (account *SCloudDelegate) GetProvider() (cloudprovider.ICloudProvider, erro
 		Secret:    passwd,
 		ProxyFunc: proxyFunc,
 
-		SApsaraEndpoints: account.Options.SApsaraEndpoints,
-		SHCSOEndpoints:   account.Options.SHCSOEndpoints,
+		Options: jsonutils.Marshal(account.Options).(*jsonutils.JSONDict),
 
 		AccountId: account.Id,
 	})

pkg/cloudid/models/cloudpolicy.go

@@ -262,7 +262,7 @@ func (self *SCloudpolicy) GetCloudgroups() ([]SCloudgroup, error) {
 	return groups, nil
 }
 
-func (self *SCloudpolicy) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SCloudpolicy) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	users, err := self.GetCloudusers()
 	if err != nil {
 		return httperrors.NewGeneralError(errors.Wrapf(err, "GetCloudusers"))
@@ -277,14 +277,14 @@ func (self *SCloudpolicy) ValidateDeleteCondition(ctx context.Context) error {
 	if len(groups) > 0 {
 		return httperrors.NewNotEmptyError("policy %s has %d groups used", self.Name, len(groups))
 	}
-	return self.SStatusInfrasResourceBase.ValidateDeleteCondition(ctx)
+	return self.SStatusInfrasResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SCloudpolicy) syncRemove(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, self)
 	defer lockman.ReleaseObject(ctx, self)
 
-	err := self.ValidateDeleteCondition(ctx)
+	err := self.ValidateDeleteCondition(ctx, nil)
 	if err != nil {
 		return err
 	}

pkg/cloudmon/collectors/apsaramon/apsaraservice.go

@@ -3,51 +3,94 @@ package apsaramon
 import (
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/apsara"
 	"yunion.io/x/onecloud/pkg/util/influxdb"
 )
 
+type SApsaraMetric struct {
+	Timestamp  int64
+	UserId     string
+	InstanceId string
+	Maximum    float64
+	Minimum    float64
+	Average    float64
+	NodeId     string
+}
+
 func (self *SApsaraCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
 	dataList := make([]influxdb.SMetricData, 0)
 	aliReg := region.(*apsara.SRegion)
 	since, until, err := common.TimeRangeFromArgs(self.Args)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "common.TimeRangeFromArgs")
 	}
-	for metricName, influxDbSpecs := range aliMetricSpecs {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "")
-		if err != nil {
-			log.Errorln(err)
+
+	instances := []api.ServerDetails{}
+	instanceMaps := map[string]api.ServerDetails{}
+	jsonutils.Update(&instances, servers)
+	for i := range instances {
+		if len(instances[i].ExternalId) == 0 {
 			continue
 		}
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, server := range servers {
-					external_id, _ := server.GetString("external_id")
-					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
-						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, metric)
-						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
+		instanceMaps[instances[i].ExternalId] = instances[i]
+		metric, err := common.FillVMCapacity(jsonutils.Marshal(instances[i]).(*jsonutils.JSONDict))
+		if err != nil {
+			return errors.Wrapf(err, "common.FillVMCapacity")
+		}
+		dataList = append(dataList, metric)
+	}
+
+	err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+	if err != nil {
+		log.Errorf("send server base metric error: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(apsaraMetricSpecs))
+	for _metricName, _influxDbSpecs := range apsaraMetricSpecs {
+		go func(metricName string, influxDbSpecs []string) {
+			defer wg.Done()
+			dataList = []influxdb.SMetricData{}
+			metricArray, err := aliReg.FetchMetricData(metricName, "acs_ecs_dashboard", since, until)
+			if err != nil {
+				log.Errorln(err)
+				return
+			}
+
+			metrics := []SApsaraMetric{}
+			jsonutils.Update(&metrics, metricArray)
+
+			for _, rtnMetric := range metrics {
+				server, ok := instanceMaps[rtnMetric.InstanceId]
+				if ok {
+					serverMetric, err := self.collectMetricFromThisServer(jsonutils.Marshal(server), rtnMetric, influxDbSpecs)
+					if err != nil {
+						log.Errorf("collect %s error: %v", metricName, err)
+						continue
 					}
+					dataList = append(dataList, serverMetric)
 				}
 			}
-		}
+			log.Infof("SendMetrics %s length: %d", metricName, len(dataList))
+			err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+			if err != nil {
+				log.Errorf("SendMetrics %s length: %d error: %v", metricName, len(dataList), err)
+			}
+
+		}(_metricName, _influxDbSpecs)
 	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+	wg.Wait()
+	return nil
 }
 
 func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
@@ -78,24 +121,25 @@ func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.
 	}
 	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
 		for _, pre := range redisDeployType {
-			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "")
+			rtnArray, err := aliReg.FetchMetricData(pre+metricName, "acs_kvstore", since, until)
 			if err != nil {
 				log.Errorln(err)
 				continue
 			}
-			if len(rtnArray) > 0 {
-				for _, rtnMetric := range rtnArray {
-					for _, server := range servers {
-						external_id, _ := server.GetString("external_id")
-						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
-							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-							node_id, _ := rtnMetric.GetString("nodeId")
-							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
-							if err != nil {
-								return err
-							}
-							dataList = append(dataList, serverMetric)
+
+			metrics := []SApsaraMetric{}
+			jsonutils.Update(&metrics, rtnArray)
+
+			for _, rtnMetric := range metrics {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if rtnMetric.InstanceId == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: rtnMetric.NodeId})
+						if err != nil {
+							return err
 						}
+						dataList = append(dataList, serverMetric)
 					}
 				}
 			}
@@ -112,30 +156,29 @@ func (self *SApsaraCloudReport) collectRegionMetricOfRds(region cloudprovider.IC
 		return err
 	}
 	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "")
+		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_rds_dashboard", since, until)
 		if err != nil {
 			log.Errorln(err)
 			continue
 		}
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, server := range servers {
-					external_id, _ := server.GetString("external_id")
-					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
-						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, metric)
-						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
+		metrics := []SApsaraMetric{}
+		jsonutils.Update(&metrics, rtnArray)
+
+		for _, rtnMetric := range metrics {
+			for _, server := range servers {
+				external_id, _ := server.GetString("external_id")
+				if rtnMetric.InstanceId == external_id {
+					metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+					if err != nil {
+						return err
 					}
-
+					dataList = append(dataList, metric)
+					serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, serverMetric)
 				}
-
 			}
 		}
 	}
@@ -150,7 +193,7 @@ func (self *SApsaraCloudReport) collectRegionMetricOfOss(region cloudprovider.IC
 		return err
 	}
 	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "")
+		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_oss", since, until)
 		if err != nil {
 			log.Errorln(err)
 			continue
@@ -182,22 +225,24 @@ func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.IC
 		return err
 	}
 	for metricName, influxDbSpecs := range aliElbMetricSpecs {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "")
+		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_slb_dashboard", since, until)
 		if err != nil {
 			log.Errorln(err)
 			continue
 		}
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, server := range servers {
-					external_id, _ := server.GetString("external_id")
-					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
-						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
+
+		metrics := []SApsaraMetric{}
+		jsonutils.Update(&metrics, rtnArray)
+
+		for _, rtnMetric := range metrics {
+			for _, server := range servers {
+				external_id, _ := server.GetString("external_id")
+				if rtnMetric.InstanceId == external_id {
+					serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+					if err != nil {
+						return err
 					}
+					dataList = append(dataList, serverMetric)
 				}
 			}
 		}
@@ -205,19 +250,13 @@ func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.IC
 	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
 }
 
-func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
-	influxDbSpecs []string) (influxdb.SMetricData, error) {
+func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric SApsaraMetric, influxDbSpecs []string) (influxdb.SMetricData, error) {
 	metric, err := self.NewMetricFromJson(server)
-	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	timestamp, _ := rtnMetric.Get("timestamp")
-	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
-	fieldValue, err := rtnMetric.Float(UNIT_AVERAGE)
 	if err != nil {
 		return influxdb.SMetricData{}, err
 	}
+	metric.Timestamp = time.Unix(rtnMetric.Timestamp/1000, 0)
+	fieldValue := rtnMetric.Average
 	//根据条件拼装metric的tag和metirc信息
 	influxDbSpec := influxDbSpecs[2]
 	measurement := common.SubstringBefore(influxDbSpec, ".")

pkg/cloudmon/collectors/apsaramon/apsarautils.go

@@ -117,6 +117,18 @@ var aliMetricSpecs = map[string][]string{
 	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
 	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
 }
+var apsaraMetricSpecs = map[string][]string{
+	"vm.MemoryUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"vm.CPUUtilization":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"InternetInRate":       {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+	"IntranetInRate":       {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"InternetOutRate":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"IntranetOutRate":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"DiskReadBPS":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBPS":         {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"DiskReadIOPS":         {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteIOPS":        {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}
 var aliRdsMetricSpecs = map[string][]string{
 	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
 	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},

pkg/cloudmon/collectors/azuremon/azureservice.go

@@ -48,7 +48,7 @@ func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.IC
 		}
 		metricNames := strings.Join(metricNameArr, ",")
 		azureReg.GetClient().Debug(true)
-		rtnMetrics, err := azureReg.GetMonitorData(metricNames, ns, external_id, since, until)
+		rtnMetrics, err := azureReg.GetMonitorData(metricNames, ns, external_id, since, until, self.Args.MetricInterval)
 		if err != nil {
 			log.Errorf("get metrics for server %s error: %v", srvPrefix, err)
 			continue

pkg/cloudmon/collectors/common/basemon.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"time"
 
 	"golang.org/x/net/http/httpproxy"
 
@@ -109,31 +110,19 @@ func (self *CloudReportBase) InitProviderInstance() (cloudprovider.ICloudProvide
 	if err != nil {
 		return nil, errors.Wrap(err, "getCloudAccount error")
 	}
-	endpoints := cloudprovider.SApsaraEndpoints{}
-	hwendpoints := cloudprovider.SHCSOEndpoints{}
 	options, err := cloudAccout.Get("options")
-	if err == nil {
-		err := options.Unmarshal(&endpoints)
-		if err != nil {
-			log.Errorf("Unmarshal SApsaraEndpoints err: %v", err)
-		}
-		err = options.Unmarshal(&hwendpoints)
-		if err != nil {
-			log.Errorf("Unmarshal SHCSOEndpoints err: %v", err)
-		}
-	} else {
-		log.Errorf("get cloudAccout options err: %v", err)
+	if err != nil {
+		log.Errorf("get cloudAccout options err:%v", err)
 	}
 	cfg := cloudprovider.ProviderConfig{
-		Id:               self.SProvider.Id,
-		Name:             self.SProvider.Name,
-		URL:              self.SProvider.AccessUrl,
-		Account:          self.SProvider.Account,
-		Secret:           secretDe,
-		Vendor:           self.SProvider.Provider,
-		ProxyFunc:        proxyFunc,
-		SApsaraEndpoints: endpoints,
-		SHCSOEndpoints:   hwendpoints,
+		Id:        self.SProvider.Id,
+		Name:      self.SProvider.Name,
+		URL:       self.SProvider.AccessUrl,
+		Account:   self.SProvider.Account,
+		Secret:    secretDe,
+		Vendor:    self.SProvider.Provider,
+		ProxyFunc: proxyFunc,
+		Options:   options.(*jsonutils.JSONDict),
 	}
 	return cloudprovider.GetProvider(cfg)
 }
@@ -218,12 +207,25 @@ func (self *CloudReportBase) CollectRegionMetric(region cloudprovider.ICloudRegi
 
 func (self *CloudReportBase) ListAllResources(manager modulebase.Manager,
 	query *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) {
+
+	return ListAllResources(manager, self.Session, query)
+}
+
+func ListAllResources(manager modulebase.Manager, session *mcclient.ClientSession,
+	query *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) {
 	offsetIndex := 0
 	resources := make([]jsonutils.JSONObject, 0)
+	tryTimes := 5
+	i := 0
 	for {
+		i++
 		query.Add(jsonutils.NewInt(int64(offsetIndex)), "offset")
-		resList, err := manager.List(self.Session, query)
+		resList, err := manager.List(session, query)
 		if err != nil {
+			if i <= tryTimes {
+				time.Sleep(3 * time.Second)
+				continue
+			}
 			return nil, err
 		}
 		resources = append(resources, resList.Data...)

pkg/cloudmon/collectors/common/multiCloudMonUtils.go

@@ -61,13 +61,14 @@ var OtherHostTag = map[string]string{
 }
 
 type ReportOptions struct {
-	Batch     int      `help:"batch"`
-	Count     int      `help:"count" json:"count"`
-	Interval  string   `help:"interval""`
-	Timeout   int64    `help:"command timeout unit:second" default:"10"`
-	SinceTime string   `help:"sinceTime"`
-	EndTime   string   `help:"endTime"`
-	Provider  []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara|HCSO" json:"provider,omitempty"`
+	Batch          int      `help:"batch"`
+	Count          int      `help:"count" json:"count"`
+	Interval       string   `help:"interval"`
+	Timeout        int64    `help:"command timeout unit:second" default:"10"`
+	SinceTime      string   `help:"sinceTime"`
+	EndTime        string   `help:"endTime"`
+	Provider       []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara|JDcloud|Ecloud|HCSO" json:"provider,omitempty"`
+	MetricInterval string   `help:"metric interval eg:PT1M"`
 	PingProbeOptions
 }
 
@@ -312,7 +313,7 @@ func TimeRangeFromArgs(args *ReportOptions) (since, until time.Time, err error)
 			return since, until, err
 		}
 	} else {
-		period64, err := strconv.ParseInt(args.Interval, 10, 8)
+		period64, err := strconv.ParseInt(args.Interval, 10, 32)
 		if err != nil {
 			return since, until, err
 		}
@@ -401,7 +402,7 @@ func SendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debu
 
 func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error {
 	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("10"), KEY_LIMIT)
 	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
 	//query.Add(jsonutils.NewString("true"), KEY_USABLE)
 	if len(args.Provider) > 0 {
@@ -409,7 +410,7 @@ func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.Clie
 			query.Add(jsonutils.NewString(val), "provider")
 		}
 	}
-	cloudProviderList, err := (&modules.Cloudproviders).List(session, query)
+	cloudProviderList, err := ListAllResources(&modules.Cloudproviders, session, query)
 	if err != nil {
 		return errors.Wrap(err, "cloudProviders get list error")
 	}
@@ -418,8 +419,8 @@ func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.Clie
 	if args.Count == 0 {
 		args.Count = 1
 	}
-	for i := 0; i < len(cloudProviderList.Data); i++ {
-		provider := cloudProviderList.Data[i]
+	for i := 0; i < len(cloudProviderList); i++ {
+		provider := cloudProviderList[i]
 		status, err := provider.GetString("status")
 		if err != nil {
 			return errors.Wrap(err, "provider get status error")

pkg/cloudnet/models/ifaces.go

@@ -106,7 +106,7 @@ func (iface *SIface) ValidateUpdateCondition(ctx context.Context) error {
 	return nil
 }
 
-func (iface *SIface) ValidateDeleteCondition(ctx context.Context) error {
+func (iface *SIface) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	// if networkid != "" {
 	// 	return errors.New("part of network, remove it by remove network memeber")
 	// }

pkg/cloudprovider/cloudprovider.go

@@ -160,8 +160,7 @@ type ProviderConfig struct {
 
 	AccountId string
 
-	SApsaraEndpoints
-	SHCSOEndpoints
+	Options *jsonutils.JSONDict
 
 	ProxyFunc httputils.TransportProxyFunc
 }

pkg/cloudprovider/extra.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudprovider
+
+type SAWSExtraOptions struct {
+	AWSAssumeRoleName string `json:"aws_assume_role_name"`
+}

pkg/cloudprovider/resources.go

@@ -787,6 +787,9 @@ type ICloudSku interface {
 
 type ICloudProject interface {
 	ICloudResource
+
+	GetDomainId() string
+	GetDomainName() string
 }
 
 type ICloudNatGateway interface {

pkg/cloudproxy/models/proxy_agents.go

@@ -79,7 +79,7 @@ func (proxyagent *SProxyAgent) ValidateUpdateData(ctx context.Context, userCred
 	return data, nil
 }
 
-func (proxyagent *SProxyAgent) ValidateDeleteCondition(ctx context.Context) error {
+func (proxyagent *SProxyAgent) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	q := ForwardManager.Query().Equals("proxy_agent_id", proxyagent.Id)
 	if count, err := q.CountWithError(); err != nil {
 		return httperrors.NewServerError("count forwards using proxy endpoint %s(%s)",

pkg/cloudproxy/models/proxy_endpoints.go

@@ -231,7 +231,7 @@ func (proxyendpoint *SProxyEndpoint) ValidateUpdateData(ctx context.Context, use
 	return input, nil
 }
 
-func (proxyendpoint *SProxyEndpoint) ValidateDeleteCondition(ctx context.Context) error {
+func (proxyendpoint *SProxyEndpoint) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	q := ForwardManager.Query().Equals("proxy_endpoint_id", proxyendpoint.Id)
 	if count, err := q.CountWithError(); err != nil {
 		return httperrors.NewServerError("count forwards using proxy endpoint %s(%s)",
@@ -244,6 +244,26 @@ func (proxyendpoint *SProxyEndpoint) ValidateDeleteCondition(ctx context.Context
 	}
 }
 
+func (proxyendpoint *SProxyEndpoint) AllowPerformPurgeForwards(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return proxyendpoint.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, proxyendpoint, "purge-forwards")
+}
+
+func (proxyendpoint *SProxyEndpoint) PerformPurgeForwards(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	q := ForwardManager.Query().Equals("proxy_endpoint_id", proxyendpoint.Id)
+	forwards := make([]SForward, 0)
+	err := db.FetchModelObjects(ForwardManager, q, &forwards)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to fetch all forwards for proxyendpoint %s", proxyendpoint.Id)
+	}
+	for i := range forwards {
+		err := forwards[i].Delete(ctx, userCred)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to delete forward %s", forwards[i].Id)
+		}
+	}
+	return nil, nil
+}
+
 func (proxyendpoint *SProxyEndpoint) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	var pms []SProxyMatch
 	q := ProxyMatchManager.Query().Equals("proxy_endpoint_id", proxyendpoint.Id)

pkg/compute/baremetal/diskconfig.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
@@ -426,6 +427,8 @@ func CheckDisksAllocable(layouts []Layout, disks []*api.DiskConfig) (bool, []*ap
 					storeFreeSize = -1
 				}
 			} else {
+				store := layouts[storeIndex]
+				log.Warningf("Disk size %dMB large than storage size %dMB, skip storage %s", disk.SizeMb, storeFreeSize, jsonutils.Marshal(store).PrettyString())
 				storeIndex++
 				storeFreeSize = -1
 			}

pkg/compute/baremetal/types.go

@@ -35,12 +35,13 @@ const (
 
 	DEFAULT_DISK_TYPE = api.DEFAULT_DISK_TYPE
 
-	DISK_DRIVER_MEGARAID   = api.DISK_DRIVER_MEGARAID
-	DISK_DRIVER_LINUX      = api.DISK_DRIVER_LINUX
-	DISK_DRIVER_HPSARAID   = api.DISK_DRIVER_HPSARAID
-	DISK_DRIVER_MPT2SAS    = api.DISK_DRIVER_MPT2SAS
-	DISK_DRIVER_MARVELRAID = api.DISK_DRIVER_MARVELRAID
-	DISK_DRIVER_PCIE       = api.DISK_DRIVER_PCIE
+	DISK_DRIVER_MEGARAID    = api.DISK_DRIVER_MEGARAID
+	DISK_DRIVER_LINUX       = api.DISK_DRIVER_LINUX
+	DISK_DRIVER_HPSARAID    = api.DISK_DRIVER_HPSARAID
+	DISK_DRIVER_MPT2SAS     = api.DISK_DRIVER_MPT2SAS
+	DISK_DRIVER_MARVELRAID  = api.DISK_DRIVER_MARVELRAID
+	DISK_DRIVER_ADAPTECRAID = api.DISK_DRIVER_ADAPTECRAID
+	DISK_DRIVER_PCIE        = api.DISK_DRIVER_PCIE
 
 	HDD_DISK_SPEC_TYPE = api.HDD_DISK_SPEC_TYPE
 	SSD_DISK_SPEC_TYPE = api.SSD_DISK_SPEC_TYPE

pkg/compute/guestdrivers/huawei.go

@@ -64,7 +64,13 @@ func (self *SHuaweiGuestDriver) GetMinimalSysDiskSizeGb() int {
 }
 
 func (self *SHuaweiGuestDriver) GetStorageTypes() []string {
-	return []string{api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS, api.STORAGE_HUAWEI_SSD}
+	return []string{
+		api.STORAGE_HUAWEI_SATA,
+		api.STORAGE_HUAWEI_SAS,
+		api.STORAGE_HUAWEI_SSD,
+		api.STORAGE_HUAWEI_GPSSD,
+		api.STORAGE_HUAWEI_ESSD,
+	}
 }
 
 func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, guest *models.SGuest, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {

pkg/compute/guestdrivers/managedvirtual.go

@@ -337,6 +337,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(ctx context.Conte
 		result.Add(jsonutils.NewBool(true), "is_running")
 	}
 
+	guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
 	return result, nil
 }
 

pkg/compute/guestdrivers/openstack.go

@@ -35,6 +35,7 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	scheduler "yunion.io/x/onecloud/pkg/scheduler/options"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
@@ -49,19 +50,19 @@ func init() {
 }
 
 func (self *SOpenStackGuestDriver) DoScheduleCPUFilter() bool {
-	return options.Options.SchedulerOptions.OpenStackSchedulerCPUFilter
+	return scheduler.GetOptions().OpenstackSchedulerCPUFilter
 }
 
 func (self *SOpenStackGuestDriver) DoScheduleMemoryFilter() bool {
-	return options.Options.SchedulerOptions.OpenStackSchedulerMemoryFilter
+	return scheduler.GetOptions().OpenstackSchedulerMemoryFilter
 }
 
 func (self *SOpenStackGuestDriver) DoScheduleSKUFilter() bool {
-	return options.Options.SchedulerOptions.OpenStackSchedulerSKUFilter
+	return scheduler.GetOptions().OpenstackSchedulerSKUFilter
 }
 
 func (self *SOpenStackGuestDriver) DoScheduleStorageFilter() bool {
-	return options.Options.SchedulerOptions.OpenStackSchedulerStorageFilter
+	return scheduler.GetOptions().OpenstackSchedulerStorageFilter
 }
 
 func (self *SOpenStackGuestDriver) GetHypervisor() string {
@@ -227,7 +228,10 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 		if err != nil {
 			return "", errors.Wrap(err, "guest.GetSystemDisk(")
 		}
-		storage, _ := sysDisk.GetStorage()
+		storage, err := sysDisk.GetStorage()
+		if err != nil {
+			return "", errors.Wrap(err, "sysDisk.GetStorage")
+		}
 		if storage.StorageType == api.STORAGE_OPENSTACK_NOVA { //不通过镜像创建磁盘的机器
 			conf := cloudprovider.SManagedVMRebuildRootConfig{
 				Account:   desc.Account,
@@ -255,11 +259,10 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 				detachDisks = append(detachDisks, iDisk.GetGlobalId())
 			}
 		}
-		defer self.attachDisks(ctx, ihost, instanceId, detachDisks)
 
-		eip, err := guest.GetElasticIp()
-		if err == nil && eip != nil {
-			ieip, err := eip.GetIEip()
+		var ieip cloudprovider.ICloudEIP
+		if eip, err := guest.GetElasticIp(); eip != nil {
+			ieip, err = eip.GetIEip()
 			if err != nil {
 				return "", errors.Wrap(err, "eip.GetIEip")
 			}
@@ -267,12 +270,10 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 			if err != nil {
 				return "", errors.Wrap(err, "ieip.Dissociate")
 			}
-			conf := &cloudprovider.AssociateConfig{
-				InstanceId:    instanceId,
-				AssociateType: api.EIP_ASSOCIATE_TYPE_SERVER,
-			}
-			defer ieip.Associate(conf)
+		} else if err != nil {
+			return "", err
 		}
+
 		err = iVM.DeleteVM(ctx)
 		if err != nil {
 			return "", errors.Wrap(err, "iVM.DeleteVM")
@@ -298,6 +299,17 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 		}
 		iVM.StopVM(ctx, opts)
 
+		if ieip != nil {
+			conf := &cloudprovider.AssociateConfig{
+				InstanceId:    instanceId,
+				AssociateType: api.EIP_ASSOCIATE_TYPE_SERVER,
+			}
+			if err = ieip.Associate(conf); err != nil {
+				return "", errors.Wrap(err, "eip.Associate")
+			}
+		}
+		self.attachDisks(ctx, ihost, instanceId, detachDisks)
+
 		iDisks, err = iVM.GetIDisks()
 		if err != nil {
 			return "", errors.Wrapf(err, "iVM.GetIDisks.AfterCreated")

pkg/compute/guestdrivers/qcloud.go

@@ -129,12 +129,12 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 	}
 
 	sysDisk := input.Disks[0]
+	if sysDisk.SizeMb < 50*1024 {
+		return nil, fmt.Errorf("The system disk size must be more than 50GB")
+	}
+
 	switch sysDisk.Backend {
-	case api.STORAGE_CLOUD_BASIC, api.STORAGE_CLOUD_SSD:
-		if sysDisk.SizeMb > 500*1024 {
-			return nil, fmt.Errorf("The %s system disk size must be less than 500GB", sysDisk.Backend)
-		}
-	case api.STORAGE_CLOUD_PREMIUM:
+	case api.STORAGE_CLOUD_BASIC, api.STORAGE_CLOUD_SSD, api.STORAGE_CLOUD_PREMIUM:
 		if sysDisk.SizeMb > 1024*1024 {
 			return nil, fmt.Errorf("The %s system disk size must be less than 1024GB", sysDisk.Backend)
 		}
@@ -150,14 +150,10 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 10GB ~ 16000GB", disk.Backend)
 			}
 		case api.STORAGE_CLOUD_PREMIUM:
-			if disk.SizeMb < 50*1024 || disk.SizeMb > 16000*1024 {
-				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 50GB ~ 16000GB", disk.Backend)
+			if disk.SizeMb < 10*1024 || disk.SizeMb > 32000*1024 {
+				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 10GB ~ 32000GB", disk.Backend)
 			}
-		case api.STORAGE_CLOUD_SSD:
-			if disk.SizeMb < 100*1024 || disk.SizeMb > 16000*1024 {
-				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", disk.Backend)
-			}
-		case api.STORAGE_CLOUD_HSSD:
+		case api.STORAGE_CLOUD_SSD, api.STORAGE_CLOUD_HSSD:
 			if disk.SizeMb < 20*1024 || disk.SizeMb > 32000*1024 {
 				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", disk.Backend)
 			}
@@ -194,14 +190,10 @@ func (self *SQcloudGuestDriver) ValidateChangeConfig(ctx context.Context, userCr
 				return httperrors.NewInputParameterError("The %s disk size must be in the range of 10GB ~ 16000GB", newDisk.Backend)
 			}
 		case api.STORAGE_CLOUD_PREMIUM:
-			if newDisk.SizeMb < 50*1024 || newDisk.SizeMb > 16000*1024 {
-				return httperrors.NewInputParameterError("The %s disk size must be in the range of 50GB ~ 16000GB", newDisk.Backend)
+			if newDisk.SizeMb < 10*1024 || newDisk.SizeMb > 32000*1024 {
+				return httperrors.NewInputParameterError("The %s disk size must be in the range of 10GB ~ 32000GB", newDisk.Backend)
 			}
-		case api.STORAGE_CLOUD_SSD:
-			if newDisk.SizeMb < 100*1024 || newDisk.SizeMb > 16000*1024 {
-				return httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", newDisk.Backend)
-			}
-		case api.STORAGE_CLOUD_HSSD:
+		case api.STORAGE_CLOUD_SSD, api.STORAGE_CLOUD_HSSD:
 			if newDisk.SizeMb < 20*1024 || newDisk.SizeMb > 32000*1024 {
 				return httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", newDisk.Backend)
 			}

pkg/compute/guestdrivers/virtualization.go

@@ -255,7 +255,29 @@ func (self *SVirtualizedGuestDriver) RequestStopGuestForDelete(ctx context.Conte
 	return nil
 }
 
+func (self *SVirtualizedGuestDriver) ValidateMachineType(machine string) error {
+	if !utils.IsInStringArray(machine, []string{api.VM_MACHINE_TYPE_PC, api.VM_MACHINE_TYPE_Q35}) {
+		return httperrors.NewBadRequestError("Invalid machine %q", machine)
+	}
+	return nil
+}
+
 func (self *SVirtualizedGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) {
+	if input.Machine != "" {
+		if err := self.ValidateMachineType(input.Machine); err != nil {
+			return nil, err
+		}
+	}
+	return input, nil
+}
+
+func (self *SVirtualizedGuestDriver) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, input *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	machine, _ := input.GetString("machine")
+	if machine != "" {
+		if err := self.ValidateMachineType(machine); err != nil {
+			return nil, err
+		}
+	}
 	return input, nil
 }
 

pkg/compute/hostdrivers/base.go

@@ -42,8 +42,8 @@ func (self *SBaseHostDriver) ValidateResetDisk(ctx context.Context, userCred mcc
 	return nil, httperrors.NewNotImplementedError("Not Implement ValidateResetDisk")
 }
 
-func (self *SBaseHostDriver) ValidateAttachStorage(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, data *jsonutils.JSONDict) error {
-	return httperrors.NewNotImplementedError("Not Implement ValidateAttachStorage")
+func (self *SBaseHostDriver) ValidateAttachStorage(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, input api.HostStorageCreateInput) (api.HostStorageCreateInput, error) {
+	return input, httperrors.NewNotImplementedError("Not Implement ValidateAttachStorage")
 }
 
 func (self *SBaseHostDriver) RequestAttachStorage(ctx context.Context, hoststorage *models.SHoststorage, host *models.SHost, storage *models.SStorage, task taskman.ITask) error {

pkg/compute/hostdrivers/huawei.go

@@ -46,7 +46,7 @@ func (self *SHuaweiHostDriver) GetHypervisor() string {
 // 系统盘必须至少40G
 func (self *SHuaweiHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error {
 	switch storage.StorageType {
-	case api.STORAGE_HUAWEI_SSD, api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS:
+	case api.STORAGE_HUAWEI_SSD, api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS, api.STORAGE_HUAWEI_ESSD, api.STORAGE_HUAWEI_GPSSD:
 		if sizeGb < 10 || sizeGb > 32768 {
 			return fmt.Errorf("The %s disk size must be in the range of 10G ~ 32768GB", storage.StorageType)
 		}

pkg/compute/hostdrivers/kvm.go

@@ -59,48 +59,47 @@ func (self *SKVMHostDriver) GetHypervisor() string {
 	return api.HYPERVISOR_KVM
 }
 
-func (self *SKVMHostDriver) ValidateAttachStorage(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, data *jsonutils.JSONDict) error {
+func (self *SKVMHostDriver) ValidateAttachStorage(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, storage *models.SStorage, input api.HostStorageCreateInput) (api.HostStorageCreateInput, error) {
 	if !utils.IsInStringArray(storage.StorageType, append([]string{api.STORAGE_LOCAL}, api.SHARED_STORAGE...)) {
-		return httperrors.NewUnsupportOperationError("Unsupport attach %s storage for %s host", storage.StorageType, host.HostType)
+		return input, httperrors.NewUnsupportOperationError("Unsupport attach %s storage for %s host", storage.StorageType, host.HostType)
 	}
 	if storage.StorageType == api.STORAGE_RBD {
 		if host.HostStatus != api.HOST_ONLINE {
-			return httperrors.NewInvalidStatusError("Attach rbd storage require host status is online")
+			return input, httperrors.NewInvalidStatusError("Attach rbd storage require host status is online")
 		}
 		pool, _ := storage.StorageConf.GetString("pool")
-		data.Set("mount_point", jsonutils.NewString(fmt.Sprintf("rbd:%s", pool)))
+		input.MountPoint = fmt.Sprintf("rbd:%s", pool)
 	} else if utils.IsInStringArray(storage.StorageType, api.SHARED_FILE_STORAGE) {
-		mountPoint, err := data.GetString("mount_point")
-		if err != nil {
-			return httperrors.NewMissingParameterError("mount_point")
+		if len(input.MountPoint) == 0 {
+			return input, httperrors.NewMissingParameterError("mount_point")
 		}
-		count, err := models.HoststorageManager.Query().Equals("host_id", host.Id).Equals("mount_point", mountPoint).CountWithError()
+		count, err := models.HoststorageManager.Query().Equals("host_id", host.Id).Equals("mount_point", input.MountPoint).CountWithError()
 		if err != nil {
-			return httperrors.NewInternalServerError("Query host storage error %s", err)
+			return input, httperrors.NewInternalServerError("Query host storage error %s", err)
 		}
 		if count > 0 {
-			return httperrors.NewBadRequestError("Host %s already have mount point %s with other storage", host.Name, mountPoint)
+			return input, httperrors.NewBadRequestError("Host %s already have mount point %s with other storage", host.Name, input.MountPoint)
 		}
 		if host.HostStatus != api.HOST_ONLINE {
-			return httperrors.NewInvalidStatusError("Attach nfs storage require host status is online")
+			return input, httperrors.NewInvalidStatusError("Attach nfs storage require host status is online")
 		}
 		if storage.StorageType == api.STORAGE_GPFS {
 			header := http.Header{}
 			header.Set(mcclient.AUTH_TOKEN, userCred.GetTokenString())
 			header.Set(mcclient.REGION_VERSION, "v2")
 			params := jsonutils.NewDict()
-			params.Set("mount_point", jsonutils.NewString(mountPoint))
+			params.Set("mount_point", jsonutils.NewString(input.MountPoint))
 			urlStr := fmt.Sprintf("%s/storages/is-mount-point?%s", host.ManagerUri, params.QueryString())
 			_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", urlStr, header, nil, false)
 			if err != nil {
-				return err
+				return input, err
 			}
 			if !jsonutils.QueryBoolean(res, "is_mount_point", false) {
-				return httperrors.NewBadRequestError("%s is not mount point %s", mountPoint, res)
+				return input, httperrors.NewBadRequestError("%s is not mount point %s", input.MountPoint, res)
 			}
 		}
 	}
-	return nil
+	return input, nil
 }
 
 func (self *SKVMHostDriver) RequestAttachStorage(ctx context.Context, hoststorage *models.SHoststorage, host *models.SHost, storage *models.SStorage, task taskman.ITask) error {
@@ -141,6 +140,7 @@ func (self *SKVMHostDriver) RequestDetachStorage(ctx context.Context, host *mode
 			mountPoint, _ := task.GetParams().GetString("mount_point")
 			body.Set("mount_point", jsonutils.NewString(mountPoint))
 			body.Set("name", jsonutils.NewString(storage.Name))
+			body.Set("storage_id", jsonutils.NewString(storage.Id))
 			_, resp, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, headers, body, false)
 			return resp, err
 		}

pkg/compute/hostdrivers/managedvirtual.go

@@ -88,8 +88,7 @@ func (self *SManagedVirtualizationHostDriver) CheckAndSetCacheImage(ctx context.
 		} else {
 			_, err = iStorageCache.GetIImageById(cachedImage.ExternalId)
 			if err != nil {
-				log.Errorf("remote image fetch error %s", err)
-				return nil, errors.Wrap(err, "iStorageCache.GetIImageById")
+				return nil, errors.Wrapf(err, "iStorageCache.GetIImageById(%s) for %s", cachedImage.ExternalId, iStorageCache.GetGlobalId())
 			}
 			image.ExternalId = cachedImage.ExternalId
 		}

pkg/compute/models/access_group_caches.go

@@ -586,14 +586,14 @@ func (manager *SAccessGroupCacheManager) Register(ctx context.Context, opts *SAc
 	return cache, cache.CreateIAccessGroup()
 }
 
-func (self *SAccessGroupCache) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SAccessGroupCache) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if self.AccessGroupId == api.DEFAULT_ACCESS_GROUP {
 		return httperrors.NewProtectedResourceError("not allow to delete default access group")
 	}
 	if self.MountTargetCount > 0 && self.Status != api.ACCESS_GROUP_STATUS_UNKNOWN {
 		return httperrors.NewNotEmptyError("access group not empty, please delete mount target first")
 	}
-	return self.SStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SStatusStandaloneResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SAccessGroupCache) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {

pkg/compute/models/access_group_rules.go

@@ -351,11 +351,11 @@ func (self *SAccessGroupRule) PreDelete(ctx context.Context, userCred mcclient.T
 	}
 }
 
-func (self *SAccessGroupRule) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SAccessGroupRule) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if self.AccessGroupId == api.DEFAULT_ACCESS_GROUP {
 		return httperrors.NewProtectedResourceError("not allow to delete default access group rule")
 	}
-	return self.SResourceBase.ValidateDeleteCondition(ctx)
+	return self.SResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (manager *SAccessGroupRuleManager) InitializeData() error {

pkg/compute/models/access_groups.go

@@ -250,7 +250,7 @@ func (self *SAccessGroup) RealDelete(ctx context.Context, userCred mcclient.Toke
 	return self.SStatusInfrasResourceBase.Delete(ctx, userCred)
 }
 
-func (self *SAccessGroup) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SAccessGroup) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if self.Id == api.DEFAULT_ACCESS_GROUP {
 		return httperrors.NewProtectedResourceError("not allow to delete default access group")
 	}
@@ -261,7 +261,7 @@ func (self *SAccessGroup) ValidateDeleteCondition(ctx context.Context) error {
 	if len(mts) > 0 {
 		return httperrors.NewNotEmptyError("access group not empty, please delete mount target first")
 	}
-	return self.SStatusInfrasResourceBase.ValidateDeleteCondition(ctx)
+	return self.SStatusInfrasResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SAccessGroup) DoSync(ctx context.Context, userCred mcclient.TokenCredential) {

pkg/compute/models/baremetalagents.go

@@ -65,18 +65,18 @@ func init() {
 	BaremetalagentManager.SetVirtualObject(BaremetalagentManager)
 }
 
-func (self *SBaremetalagent) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SBaremetalagent) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if self.Status == api.BAREMETAL_AGENT_ENABLED {
 		return fmt.Errorf("Cannot delete in status %s", self.Status)
 	}
 	storageCache, _ := self.getStorageCache()
 	if storageCache != nil {
-		err := storageCache.ValidateDeleteCondition(ctx)
+		err := storageCache.ValidateDeleteCondition(ctx, nil)
 		if err != nil {
 			return fmt.Errorf("storagecache cannot be delete: %s", err)
 		}
 	}
-	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SBaremetalagent) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.BaremetalagentUpdateInput) (api.BaremetalagentUpdateInput, error) {

pkg/compute/models/buckets.go

@@ -1180,12 +1180,12 @@ func (bucket *SBucket) PerformSync(
 }
 
 func (bucket *SBucket) ValidatePurgeCondition(ctx context.Context) error {
-	return bucket.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
+	return bucket.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
-func (bucket *SBucket) ValidateDeleteCondition(ctx context.Context) error {
+func (bucket *SBucket) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if bucket.Status == api.BUCKET_STATUS_UNKNOWN {
-		return bucket.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
+		return bucket.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, nil)
 	}
 	if bucket.ObjectCnt > 0 {
 		return httperrors.NewNotEmptyError("Buckets that are not empty do not support this operation")

pkg/compute/models/cachedimages.go

@@ -37,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -101,7 +102,7 @@ func (self SCachedimage) GetGlobalId() string {
 	return self.ExternalId
 }
 
-func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	cnt, err := self.getStoragecacheCount()
 	if err != nil {
 		return httperrors.NewInternalServerError("ValidateDeleteCondition error %s", err)
@@ -112,7 +113,7 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {
 	if self.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !self.isReferenceSessionExpire() {
 		return httperrors.NewConflictError("the image reference session has not been expired!")
 	}
-	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx)
+	return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SCachedimage) isReferenceSessionExpire() bool {
@@ -231,11 +232,6 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	imageCache := SCachedimage{}
 	imageCache.SetModelManager(manager, &imageCache)
 
-	img.Name, err = db.GenerateName(manager, nil, img.Name)
-	if err != nil {
-		return nil, errors.Wrapf(err, "db.GenerateName(%s)", img.Name)
-	}
-
 	err = manager.RawQuery().Equals("id", img.Id).First(&imageCache)
 	if err != nil {
 		if err == sql.ErrNoRows { // insert
@@ -313,7 +309,6 @@ func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred m
 	s := auth.GetAdminSession(ctx, options.Options.Region, "")
 	obj, err := modules.Images.Get(s, imageId, nil)
 	if err != nil {
-		log.Errorf("GetImageById %s error %s", imageId, err)
 		return nil, errors.Wrap(err, "modules.Images.Get")
 	}
 	cachedImage, err := manager.cacheGlanceImageInfo(ctx, userCred, obj)
@@ -820,6 +815,40 @@ func (manager *SCachedimageManager) ListItemExportKeys(ctx context.Context, q *s
 	return q, nil
 }
 
+// 清理已经删除的镜像缓存
+func (manager *SCachedimageManager) AutoCleanImageCaches(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	lastSync := time.Now().Add(time.Duration(-1*api.CACHED_IMAGE_REFERENCE_SESSION_EXPIRE_SECONDS) * time.Second)
+	q := manager.Query()
+	q = q.LT("last_sync", lastSync).Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE).IsNullOrEmpty("external_id").Limit(50)
+	caches := []SCachedimage{}
+	err := db.FetchModelObjects(manager, q, &caches)
+	if err != nil {
+		return
+	}
+	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	for i := range caches {
+		_, err := modules.Images.Get(s, caches[i].Id, nil)
+		if err != nil {
+			if e, ok := err.(*httputils.JSONClientError); ok && e.Code == 404 {
+				e := caches[i].ValidateDeleteCondition(ctx, nil)
+				if e == nil {
+					caches[i].Delete(ctx, userCred)
+					continue
+				}
+				db.Update(&caches[i], func() error {
+					caches[i].Name = fmt.Sprintf("%s-deleted@%s", caches[i].Name, timeutils.ShortDate(time.Now()))
+					return nil
+				})
+			}
+			continue
+		}
+		db.Update(&caches[i], func() error {
+			caches[i].LastSync = time.Now()
+			return nil
+		})
+	}
+}
+
 func (manager *SCachedimageManager) InitializeData() error {
 	images := []SCachedimage{}
 	q := manager.Query().IsNullOrEmpty("tenant_id")

pkg/compute/models/cloudaccounts.go

@@ -203,7 +203,7 @@ func (self *SCloudaccount) getCloudprovidersInternal(enabled tristate.TriState)
 	return cloudproviders
 }
 
-func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	// allow delete cloudaccount if it is disabled
 	// if self.EnableAutoSync {
 	//	return httperrors.NewInvalidStatusError("automatic syncing is enabled")
@@ -216,12 +216,12 @@ func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error {
 	}
 	cloudproviders := self.GetCloudproviders()
 	for i := 0; i < len(cloudproviders); i++ {
-		if err := cloudproviders[i].ValidateDeleteCondition(ctx); err != nil {
+		if err := cloudproviders[i].ValidateDeleteCondition(ctx, nil); err != nil {
 			return httperrors.NewInvalidStatusError("provider %s: %v", cloudproviders[i].Name, err)
 		}
 	}
 
-	return self.SEnabledStatusInfrasResourceBase.ValidateDeleteCondition(ctx)
+	return self.SEnabledStatusInfrasResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (self *SCloudaccount) enableAccountOnly(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) {
@@ -437,15 +437,11 @@ func (manager *SCloudaccountManager) validateCreateData(
 	}
 
 	var endpointOptions jsonutils.JSONObject
-	endpoints := cloudprovider.SApsaraEndpoints{}
 	if input.SCloudaccountCredential.SApsaraEndpoints != nil {
-		endpoints = *input.SCloudaccountCredential.SApsaraEndpoints
 		endpointOptions = jsonutils.Marshal(input.SCloudaccountCredential.SApsaraEndpoints)
 	}
 
-	hcsoEndpoints := cloudprovider.SHCSOEndpoints{}
 	if input.SCloudaccountCredential.SHCSOEndpoints != nil {
-		hcsoEndpoints = *input.SCloudaccountCredential.SHCSOEndpoints
 		endpointOptions = jsonutils.Marshal(input.SCloudaccountCredential.SHCSOEndpoints)
 	}
 
@@ -511,8 +507,7 @@ func (manager *SCloudaccountManager) validateCreateData(
 		Secret:    input.Secret,
 		ProxyFunc: proxyFunc,
 
-		SApsaraEndpoints: endpoints,
-		SHCSOEndpoints:   hcsoEndpoints,
+		Options: input.Options,
 	})
 	if err != nil {
 		if err == cloudprovider.ErrNoSuchProvder {
@@ -740,12 +735,12 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 	}
 
 	accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
-		Vendor:         self.Provider,
-		URL:            self.AccessUrl,
-		Account:        account.Account,
-		Secret:         account.Secret,
-		SHCSOEndpoints: hcsoEndpoints,
-		ProxyFunc:      self.proxyFunc(),
+		Vendor:    self.Provider,
+		URL:       self.AccessUrl,
+		Account:   account.Account,
+		Secret:    account.Secret,
+		Options:   self.Options,
+		ProxyFunc: self.proxyFunc(),
 	})
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error())
@@ -936,15 +931,6 @@ func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider,
 	if err != nil {
 		return nil, fmt.Errorf("Invalid password %s", err)
 	}
-	endpoints := cloudprovider.SApsaraEndpoints{}
-	hcsoEndpoints := cloudprovider.SHCSOEndpoints{}
-	if self.Options != nil {
-		if self.Provider == api.CLOUD_PROVIDER_HCSO {
-			self.Options.Unmarshal(&hcsoEndpoints)
-		} else if self.Provider == api.CLOUD_PROVIDER_APSARA {
-			self.Options.Unmarshal(&endpoints)
-		}
-	}
 	return cloudprovider.GetProvider(cloudprovider.ProviderConfig{
 		Id:      self.Id,
 		Name:    self.Name,
@@ -953,9 +939,8 @@ func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider,
 		Account: self.Account,
 		Secret:  secret,
 
-		SApsaraEndpoints: endpoints,
-		SHCSOEndpoints:   hcsoEndpoints,
-		ProxyFunc:        self.proxyFunc(),
+		Options:   self.Options,
+		ProxyFunc: self.proxyFunc(),
 	})
 }
 
@@ -1218,7 +1203,7 @@ func (self *SCloudaccount) GetEnvironment() string {
 	return self.AccessUrl
 }
 
-func (self *SCloudaccount) getMoreDetails(out api.CloudaccountDetail) api.CloudaccountDetail {
+func (self *SCloudaccount) getMoreDetails(ctx context.Context, out api.CloudaccountDetail) api.CloudaccountDetail {
 	out.EipCount, _ = self.GetEipCount()
 	out.VpcCount, _ = self.GetVpcCount()
 	out.DiskCount, _ = self.GetDiskCount()
@@ -1231,7 +1216,7 @@ func (self *SCloudaccount) getMoreDetails(out api.CloudaccountDetail) api.Clouda
 
 	out.Projects = []api.ProviderProject{}
 	for _, projectId := range self.getProjectIds() {
-		if proj, _ := db.TenantCacheManager.FetchTenantById(context.Background(), projectId); proj != nil {
+		if proj, _ := db.TenantCacheManager.FetchTenantById(ctx, projectId); proj != nil {
 			project := api.ProviderProject{
 				Tenant:   proj.Name,
 				TenantId: proj.Id,
@@ -1298,7 +1283,7 @@ func (manager *SCloudaccountManager) FetchCustomizeColumns(
 			detail.ProxySetting.HTTPSProxy = proxySetting.HTTPSProxy
 			detail.ProxySetting.NoProxy = proxySetting.NoProxy
 		}
-		rows[i] = account.getMoreDetails(detail)
+		rows[i] = account.getMoreDetails(ctx, detail)
 	}
 
 	return rows
@@ -2154,6 +2139,16 @@ func (self *SCloudaccount) RealDelete(ctx context.Context, userCred mcclient.Tok
 			return errors.Wrapf(err, "project %s Delete", projects[i].Id)
 		}
 	}
+	caches, err := self.GetDnsZoneCaches()
+	if err != nil {
+		return errors.Wrapf(err, "GetDnsZoneCaches")
+	}
+	for i := range caches {
+		err = caches[i].RealDelete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "dns zone cache %s delete", caches[i].Id)
+		}
+	}
 	return self.SEnabledStatusInfrasResourceBase.Delete(ctx, userCred)
 }
 
@@ -2805,6 +2800,13 @@ func (self *SCloudaccount) SyncDnsZones(ctx context.Context, userCred mcclient.T
 			result.UpdateError(errors.Wrapf(err, "SyncWithCloudDnsZone"))
 			continue
 		}
+		zone, err := commondb[i].GetDnsZone()
+		if err != nil {
+			result.UpdateError(errors.Wrapf(err, "GetDnsZone"))
+			continue
+		}
+		localZones = append(localZones, *zone)
+		remoteZones = append(remoteZones, commonext[i])
 
 		result.Update()
 	}
@@ -2816,10 +2818,10 @@ func (self *SCloudaccount) SyncDnsZones(ctx context.Context, userCred mcclient.T
 			continue
 		}
 		if isNew {
-			localZones = append(localZones, *dnsZone)
-			remoteZones = append(remoteZones, added[i])
+			result.Add()
 		}
-		result.Add()
+		localZones = append(localZones, *dnsZone)
+		remoteZones = append(remoteZones, added[i])
 	}
 
 	return localZones, remoteZones, result

pkg/compute/models/cloudimages.go

@@ -84,7 +84,7 @@ func (self *SCloudimage) syncRemove(ctx context.Context, userCred mcclient.Token
 	}
 	image := _image.(*SCachedimage)
 
-	err = image.ValidateDeleteCondition(ctx)
+	err = image.ValidateDeleteCondition(ctx, nil)
 	if err == nil {
 		image.Delete(ctx, userCred)
 	}

pkg/compute/models/cloudproviderregions.go

@@ -171,7 +171,7 @@ func (manager *SCloudproviderregion) ValidateCreateData(ctx context.Context, use
 	return nil, httperrors.NewForbiddenError("not allow to create")
 }
 
-func (self *SCloudproviderregion) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SCloudproviderregion) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	return nil
 }
 

pkg/compute/models/cloudproviders.go

@@ -173,7 +173,7 @@ func (self *SCloudprovider) GetProjectMapping() (*SProjectMapping, error) {
 	return cache.GetProjectMapping()
 }
 
-func (self *SCloudprovider) ValidateDeleteCondition(ctx context.Context) error {
+func (self *SCloudprovider) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	// allow delete cloudprovider if it is disabled
 	// account := self.GetCloudaccount()
 	// if account != nil && account.EnableAutoSync {
@@ -189,7 +189,7 @@ func (self *SCloudprovider) ValidateDeleteCondition(ctx context.Context) error {
 	// if !usage.isEmpty() {
 	// 	return httperrors.NewNotEmptyError("Not an empty cloud provider")
 	// }
-	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx, nil)
 }
 
 func (manager *SCloudproviderManager) GetPublicProviderIdsQuery() *sqlchemy.SSubQuery {
@@ -419,20 +419,23 @@ func createTenant(ctx context.Context, name, domainId, desc string) (string, str
 	return domainId, projectId, nil
 }
 
-func (self *SCloudaccount) getOrCreateTenant(ctx context.Context, name, projectId, desc string) (string, string, error) {
+func (self *SCloudaccount) getOrCreateTenant(ctx context.Context, name, domainId, projectId, desc string) (string, string, error) {
+	if len(domainId) == 0 {
+		domainId = self.DomainId
+	}
 	tenant, err := getTenant(ctx, projectId, name)
 	if err != nil {
 		if errors.Cause(err) != sql.ErrNoRows {
 			return "", "", errors.Wrapf(err, "getTenan")
 		}
-		return createTenant(ctx, name, self.DomainId, desc)
+		return createTenant(ctx, name, domainId, desc)
 	}
 	share := self.GetSharedInfo()
 	if tenant.DomainId == self.DomainId || (share.PublicScope == rbacutils.ScopeSystem ||
 		(share.PublicScope == rbacutils.ScopeDomain && utils.IsInStringArray(tenant.DomainId, share.SharedDomains))) {
 		return tenant.DomainId, tenant.Id, nil
 	}
-	return createTenant(ctx, name, self.DomainId, desc)
+	return createTenant(ctx, name, domainId, desc)
 }
 
 func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.TokenCredential) error {
@@ -442,7 +445,7 @@ func (self *SCloudprovider) syncProject(ctx context.Context, userCred mcclient.T
 	}
 
 	desc := fmt.Sprintf("auto create from cloud provider %s (%s)", self.Name, self.Id)
-	domainId, projectId, err := account.getOrCreateTenant(ctx, self.Name, self.ProjectId, desc)
+	domainId, projectId, err := account.getOrCreateTenant(ctx, self.Name, "", self.ProjectId, desc)
 	if err != nil {
 		return errors.Wrap(err, "getOrCreateTenant")
 	}
@@ -874,16 +877,6 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error)
 
 	account := self.GetCloudaccount()
 
-	endpoints := cloudprovider.SApsaraEndpoints{}
-	hscsoEndpoints := cloudprovider.SHCSOEndpoints{}
-	if account.Options != nil {
-		if self.Provider == api.CLOUD_PROVIDER_HCSO {
-			account.Options.Unmarshal(&hscsoEndpoints)
-		} else if self.Provider == api.CLOUD_PROVIDER_APSARA {
-			account.Options.Unmarshal(&endpoints)
-		}
-	}
-
 	return cloudprovider.GetProvider(cloudprovider.ProviderConfig{
 		Id:        self.Id,
 		Name:      self.Name,
@@ -893,8 +886,7 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error)
 		Secret:    passwd,
 		ProxyFunc: account.proxyFunc(),
 
-		SApsaraEndpoints: endpoints,
-		SHCSOEndpoints:   hscsoEndpoints,
+		Options: account.Options,
 	})
 }