Merge pull request #2527 from ioito/automated-cherry-pick-of-#2522-upstream-release-2.8.0

Automated cherry pick of #2522: fix: sync status after guest sync conf

.circleci/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7
+MAINTAINER Yousong Zhou <zhouyousong@yunion.cn>
+
+RUN true \
+	&& yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
+	&& yum install -y git wget \
+	&& yum groupinstall -y "Development Tools" \
+	&& : ceph for cmd/host \
+	&& rpm --import 'https://download.ceph.com/keys/release.asc' \
+	&& yum install -y https://download.ceph.com/rpm-luminous/el7/noarch/ceph-release-1-1.el7.noarch.rpm \
+	&& yum install -y libcephfs-devel librbd-devel librados-devel \
+	&& yum clean all
+
+WORKDIR /opt
+RUN true \
+	&& wget https://dl.google.com/go/go1.12.1.linux-amd64.tar.gz \
+	&& tar xzf go1.12.1.linux-amd64.tar.gz \
+	&& rm -vf  go1.12.1.linux-amd64.tar.gz \
+	&& true
+ENV GOROOT="/opt/go"
+ENV PATH="/opt/go/bin:${PATH}"
+
+RUN useradd -c "OneCloud Builder" --create-home --home-dir /home/build --shell /bin/bash build
+USER build
+ENV HOME /home/build
+WORKDIR /home/build
+
+ENV GOPATH="/home/build/go"
+ENV PATH="${GOPATH}/bin:${PATH}"

.circleci/config.yml

@@ -0,0 +1,29 @@
+# Golang CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-go/ for more details
+version: 2
+jobs:
+  build:
+    docker:
+      - image: yunionio/onecloud-ci:20190328
+
+    #### TEMPLATE_NOTE: go expects specific checkout path representing url
+    #### expecting it in the form of
+    ####   /go/src/github.com/circleci/go-tool
+    ####   /go/src/bitbucket.org/circleci/go-tool
+    working_directory: /home/build/go/src/yunion.io/x/onecloud
+    parallelism: 4
+    steps:
+      - checkout
+      - run:
+          command: |
+            targets="$(circleci tests glob "cmd/*" | grep -v cmd/host-image | circleci tests split)"
+            echo $targets | tr ' ' '\n'
+            make $targets
+            ls -lh _output/bin/
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - build

.travis.yml

@@ -0,0 +1,22 @@
+# This is how to tell Travis to use the fast container-based test
+# runner instead of the slow VM-based one.
+sudo: false
+
+dist: xenial
+
+services:
+  - docker
+
+language: go
+
+go_import_path: yunion.io/x/onecloud
+
+go:
+  - "1.12.x"
+
+# Only clone the most recent commit.
+git:
+  depth: 1
+
+script:
+  - make docker_build

Gopkg.lock

@@ -524,6 +524,14 @@
   revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
   version = "v1.4.7"
 
+[[projects]]
+  digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda"
+  name = "github.com/ghodss/yaml"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
+  version = "v1.0.0"
+
 [[projects]]
   branch = "master"
   digest = "1:36fe9527deed01d2a317617e59304eb2c4ce9f8a24115bcc5c2e37b3aee5bae4"
@@ -533,9 +541,10 @@
   revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
 
 [[projects]]
-  digest = "1:ff09f03ca12c4d23c28c596eb0738745e36c1cac111f62f64819210f40607cd2"
+  digest = "1:d5083934eb25e45d17f72ffa86cae3814f4a9d6c073c4f16b64147169b245606"
   name = "github.com/gin-gonic/gin"
   packages = [
+    ".",
     "binding",
     "json",
     "render",
@@ -851,6 +860,14 @@
   revision = "db8e3cd836b82e82e0a9c8edc6896967dd31374f"
   version = "v1.1.3"
 
+[[projects]]
+  branch = "master"
+  digest = "1:74e674486930998805335f2b2f0af5fd3b183c755973e53f532363a56831b7a3"
+  name = "github.com/libvirt/libvirt-go-xml"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "4b6d354bfe8a0c85cebc68f0f7e45f7d5a37de7b"
+
 [[projects]]
   digest = "1:4e878df5f4e9fd625bf9c9aac77ef7cbfa4a74c01265505527c23470c0e40300"
   name = "github.com/marstr/guid"
@@ -1063,12 +1080,12 @@
   version = "v2.0.7"
 
 [[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
+  branch = "master"
+  digest = "1:deff41c4160cc5b487a1d1a69a58925f2d3ac11befa0692b50e76940594e20f5"
   name = "github.com/pkg/errors"
   packages = ["."]
   pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
+  revision = "27936f6d90f9c8e1145f11ed52ffffbfdb9e0af7"
 
 [[projects]]
   branch = "master"
@@ -1211,9 +1228,20 @@
   version = "v1.0.3"
 
 [[projects]]
-  digest = "1:18752d0b95816a1b777505a97f71c7467a8445b8ffb55631a7bf779f6ba4fa83"
+  digest = "1:ac83cf90d08b63ad5f7e020ef480d319ae890c208f8524622a2f3136e2686b02"
+  name = "github.com/stretchr/objx"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "477a77ecc69700c7cdeb1fa9e129548e1c1c393c"
+  version = "v0.1.1"
+
+[[projects]]
+  digest = "1:15a4a7e5afac3cea801fa24831fce3bf3b5bd3620cbf8355a07b7dbf06877883"
   name = "github.com/stretchr/testify"
-  packages = ["assert"]
+  packages = [
+    "assert",
+    "mock",
+  ]
   pruneopts = "UT"
   revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
   version = "v1.2.2"
@@ -1476,14 +1504,6 @@
   revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8"
   version = "v1.17.0"
 
-[[projects]]
-  digest = "1:2655728699af00d823b0a68bd7169fd241a30c04252f456b34421908701b9c3f"
-  name = "gopkg.in/gin-gonic/gin.v1"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
-  version = "v1.2"
-
 [[projects]]
   digest = "1:cbc72c4c4886a918d6ab4b95e347ffe259846260f99ebdd8a198c2331cf2b2e9"
   name = "gopkg.in/go-playground/validator.v8"
@@ -1684,28 +1704,29 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:5ef22fef33e8bfb5e66901f9adba50f5d594ad20263a132fdc62801033eb7712"
+  digest = "1:f044fe02305a360710bcb0626b870599537514da95c9d4f4cd31df0b4c017e5f"
   name = "yunion.io/x/jsonutils"
   packages = ["."]
   pruneopts = "UT"
-  revision = "ff8561d72c32ba4eb91ee660f6f2ca9e6f64d268"
+  revision = "a964e1e8a05199146284389fe6ef2d1eac2744b4"
 
 [[projects]]
   branch = "master"
-  digest = "1:636db180d3fc734536a5437d93bb6fe937feb036c1ed062c25872a47c4283163"
+  digest = "1:a872813eff1f3d353a1e4e64d745ac07387ed52cbc4cc2f6180272a7df7f21d1"
   name = "yunion.io/x/log"
   packages = [
     ".",
     "hooks",
   ]
   pruneopts = "UT"
-  revision = "5185c49f6d361f61e561fbccba1bedcc7c62748b"
+  revision = "9f6483a7103dc7b5012ae7aad2c74728e02cf650"
 
 [[projects]]
   branch = "master"
-  digest = "1:f1f1aa2c29c1de090cf90d661fa1ee6c45c2d828059b5ec265a88a771649f83b"
+  digest = "1:c09298fb61e83dae28b08222e20f31bd109f93c50a29517a17edcc5a5666bd1a"
   name = "yunion.io/x/pkg"
   packages = [
+    "errors",
     "gotypes",
     "prettytable",
     "trace",
@@ -1730,30 +1751,29 @@
     "util/stringutils",
     "util/timeutils",
     "util/trace",
-    "util/ttlpool",
     "util/version",
     "util/wait",
     "util/workqueue",
     "utils",
   ]
   pruneopts = "UT"
-  revision = "f92993f80cd49639f283ff29d6ab3d7cdd6ca941"
+  revision = "b564cfdcc2247b3f6f6bc4e818b44f08d82dc660"
 
 [[projects]]
   branch = "master"
-  digest = "1:4137f40ed548a5dc02719ad2e6b6bdfd76a9d881fcd1a72eb5970956a7219d03"
+  digest = "1:f7386c58a1e94eddbca9b8a95167ba7b4bc7d8dfea40a01261f7443757eafb25"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
   pruneopts = "UT"
-  revision = "4d8d8e9b16804336f16570787421b9be24a6da1f"
+  revision = "cfa59aeabbfe47c910920a2e1d61e1eaddd92ec5"
 
 [[projects]]
   branch = "master"
-  digest = "1:6a0bb09937441c871c34278e62ea9c80320a6b9894d63261f5c4246c07aeb510"
+  digest = "1:d91515edf1e995676f635f39ed5445e45fd3d176c71546744b11cba7e0126421"
   name = "yunion.io/x/structarg"
   packages = ["."]
   pruneopts = "UT"
-  revision = "f370514c4558e64e798bd83abaff263dabef9f3a"
+  revision = "115bed041de33bdf3a5be5281d4a6b06607c82d8"
 
 [solve-meta]
   analyzer-name = "dep"
@@ -1806,7 +1826,6 @@
     "github.com/coredns/coredns/plugin/nsid",
     "github.com/coredns/coredns/plugin/pkg/dnsutil",
     "github.com/coredns/coredns/plugin/pkg/fall",
-    "github.com/coredns/coredns/plugin/pkg/log",
     "github.com/coredns/coredns/plugin/pkg/upstream",
     "github.com/coredns/coredns/plugin/proxy",
     "github.com/coredns/coredns/plugin/reload",
@@ -1814,6 +1833,7 @@
     "github.com/coredns/coredns/request",
     "github.com/fatih/color",
     "github.com/fsnotify/fsnotify",
+    "github.com/gin-gonic/gin",
     "github.com/go-sql-driver/mysql",
     "github.com/golang-plus/errors",
     "github.com/golang-plus/uuid",
@@ -1827,12 +1847,14 @@
     "github.com/json-iterator/go",
     "github.com/koding/websocketproxy",
     "github.com/kr/pty",
+    "github.com/libvirt/libvirt-go-xml",
     "github.com/mholt/caddy",
     "github.com/mholt/caddy/startupshutdown",
     "github.com/miekg/dns",
     "github.com/moul/http2curl",
     "github.com/nelsonken/cos-go-sdk-v5/cos",
     "github.com/pierrec/lz4",
+    "github.com/pkg/errors",
     "github.com/serialx/hashring",
     "github.com/shirou/gopsutil/cpu",
     "github.com/shirou/gopsutil/host",
@@ -1840,6 +1862,7 @@
     "github.com/shirou/gopsutil/net",
     "github.com/shirou/gopsutil/process",
     "github.com/stretchr/testify/assert",
+    "github.com/stretchr/testify/mock",
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common",
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors",
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http",
@@ -1858,7 +1881,6 @@
     "golang.org/x/net/bpf",
     "golang.org/x/net/ipv4",
     "golang.org/x/sys/unix",
-    "gopkg.in/gin-gonic/gin.v1",
     "gopkg.in/yaml.v2",
     "k8s.io/api/core/v1",
     "k8s.io/apimachinery/pkg/api/errors",
@@ -1871,6 +1893,8 @@
     "k8s.io/kubernetes/pkg/scheduler/api/v1",
     "yunion.io/x/jsonutils",
     "yunion.io/x/log",
+    "yunion.io/x/log/hooks",
+    "yunion.io/x/pkg/errors",
     "yunion.io/x/pkg/gotypes",
     "yunion.io/x/pkg/prettytable",
     "yunion.io/x/pkg/trace",
@@ -1893,7 +1917,6 @@
     "yunion.io/x/pkg/util/stringutils",
     "yunion.io/x/pkg/util/timeutils",
     "yunion.io/x/pkg/util/trace",
-    "yunion.io/x/pkg/util/ttlpool",
     "yunion.io/x/pkg/util/version",
     "yunion.io/x/pkg/util/wait",
     "yunion.io/x/pkg/util/workqueue",

Gopkg.toml

@@ -206,4 +206,7 @@
 
 [[constraint]]
   name="github.com/ceph/go-ceph"
-  branch = "master"
+  branch = "master"
+[[constraint]]
+  branch = "master"
+  name = "github.com/libvirt/libvirt-go-xml"

Makefile

@@ -117,3 +117,6 @@ dep:
 
 %:
 	@:
+
+docker_build:
+	$(ROOT_DIR)/build/docker_build.sh

OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+reviewers:
+  - swordqiu
+  - yousong
+  - wanyaoqi
+  - Zexi
+  - ioito
+  - tb365
+approvers:
+  - swordqiu
+  - yousong
+  - wanyaoqi
+  - Zexi

README.md

@@ -1 +1,4 @@
-# Yunion onecloud
+# Yunion OneCloud
+
+[![CircleCI](https://circleci.com/gh/yunionio/onecloud.svg?style=svg)](https://circleci.com/gh/yunionio/onecloud) [![Build Status](https://travis-ci.org/yunionio/onecloud.svg?branch=master)](https://travis-ci.org/yunionio/onecloud)
+[![Go Report Card](https://goreportcard.com/badge/github.com/yunionio/onecloud)](https://goreportcard.com/report/github.com/yunionio/onecloud)

build/awscli/vars

@@ -0,0 +1 @@
+DESCRIPTION="AWS Command Line Utility"

build/azurecli/vars

@@ -0,0 +1 @@
+DESCRIPTION="Azure Command Line Utility"

build/baremetal-agent/vars

@@ -0,0 +1 @@
+DESCRIPTION="Baremetal Agent Command Line Utility"

build/docker_build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SRC_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
+BIN_DIR=$SRC_ROOT/_output/bin
+WORK_DIR="/go/src/yunion.io/x/onecloud"
+
+docker run --rm -it \
+    -w $WORK_DIR \
+    -v $SRC_ROOT:$WORK_DIR \
+    d3lx/golang:yunion \
+    make
+
+echo "Build finish"
+ls -alh $BIN_DIR

build/esxicli/vars

@@ -0,0 +1 @@
+DESCRIPTION="ESXI Command Line Utility"

build/host/vars

@@ -0,0 +1 @@
+DESCRIPTION="Yunion Host Command Line Utility"

build/huaweicli/vars

@@ -0,0 +1 @@
+DESCRIPTION="HUAWEI Command Line Utility"

build/ipmicli/vars

@@ -0,0 +1 @@
+DESCRIPTION="IPMI Command Line Utility"

build/openstackcli/vars

@@ -0,0 +1 @@
+DESCRIPTION="OPENSTACK Command Line Utility"

build/qcloudcli/vars

@@ -0,0 +1 @@
+DESCRIPTION="QCLOUD Command Line Utility"

build/region/root/opt/yunion/share/notify_templates/compute/content/SERVER_REBUILD_ROOT

@@ -1 +1 @@
-您的{{ .hypervisor }}主机{{ .name }}的系统盘已经重置成功，{{ if .account }}初始帐号为{{ .account }}，{{ end }}{{ if .keypair }}访问密钥为{{ .keypair }}，{{ end }}{{ if .password }}初始密码为{{ .password }}，{{ end }}请使用{{ if .windows }}远程桌面连接器(RDC){{ else }}SSH{{ end }}或控制面板控制台访问云主机。
+您的{{ .hypervisor }}主机{{ .name }}重装系统成功，{{ if .account }}初始帐号为{{ .account }}，{{ end }}{{ if .keypair }}访问密钥为{{ .keypair }}，{{ end }}{{ if .password }}初始密码为{{ .password }}，{{ end }}请使用{{ if .windows }}远程桌面连接器(RDC){{ else }}SSH{{ end }}或控制面板控制台访问云主机。

build/region/root/opt/yunion/share/notify_templates/compute/title/SERVER_REBUILD_ROOT

@@ -1 +1 @@
-{{ .hypervisor }}主机{{ .name }}的系统盘重置成功
+{{ .hypervisor }}主机{{ .name }}重装系统成功

cmd/baremetal-agent/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/baremetal/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.New().StartService()
 }

cmd/climc/shell/actions.go

@@ -1,6 +1,8 @@
 package shell
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -17,6 +19,8 @@ type BaseActionListOptions struct {
 	Action     []string `help:"Log action"`
 	Search     string   `help:"Filter action logs by obj_name, using 'like' syntax."`
 	Admin      bool     `help:"admin mode"`
+	Succ       bool     `help:"Show success action log only"`
+	Fail       bool     `help:"Show failed action log only"`
 }
 
 type ActionListOptions struct {
@@ -64,6 +68,15 @@ func doActionList(s *mcclient.ClientSession, args *ActionListOptions) error {
 	if args.Admin {
 		params.Add(jsonutils.JSONTrue, "admin")
 	}
+	if args.Succ && args.Fail {
+		return fmt.Errorf("succ and fail can't go together")
+	}
+	if args.Succ {
+		params.Add(jsonutils.JSONTrue, "success")
+	}
+	if args.Fail {
+		params.Add(jsonutils.JSONFalse, "success")
+	}
 	logs, err := modules.Actions.List(s, params)
 	if err != nil {
 		return err

cmd/climc/shell/events.go

@@ -38,6 +38,10 @@ func doComputeEventList(s *mcclient.ClientSession, args *EventListOptions) error
 	return doEventList(modules.Logs, s, args)
 }
 
+func doImageEventList(s *mcclient.ClientSession, args *EventListOptions) error {
+	return doEventList(modules.ImageLogs, s, args)
+}
+
 func doEventList(man modules.ResourceManager, s *mcclient.ClientSession, args *EventListOptions) error {
 	params := jsonutils.NewDict()
 	if len(args.Type) > 0 {
@@ -144,4 +148,9 @@ func init() {
 		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"kubemachine"}}
 		return doK8sEventList(s, &nargs)
 	})
+
+	R(&TypeEventListOptions{}, "image-event", "Show operation event logs of glance images", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"image"}}
+		return doImageEventList(s, &nargs)
+	})
 }

cmd/climc/shell/hosts.go

@@ -32,6 +32,10 @@ func init() {
 
 		Usable *bool `help:"List all zones that is usable"`
 
+		Hypervisor string `help:"filter hosts by hypervisor"`
+
+		StorageNotAttached bool `help:"List hosts not attach specified storage"`
+
 		options.BaseListOptions
 	}
 	R(&HostListOptions{}, "host-list", "List hosts", func(s *mcclient.ClientSession, opts *HostListOptions) error {
@@ -449,6 +453,7 @@ func init() {
 		MAC        string `help:"Default MAC address of baremetal"`
 		Rack       string `help:"Rack number of baremetal"`
 		Slots      string `help:"Slots number of baremetal"`
+		IpmiUser   string `help:"IPMI user name"`
 		IpmiPasswd string `help:"IPMI user password"`
 		IpmiAddr   string `help:"IPMI IP address"`
 	}
@@ -463,6 +468,9 @@ func init() {
 		if len(args.Slots) > 0 {
 			params.Add(jsonutils.NewString(args.Slots), "slots")
 		}
+		if len(args.IpmiUser) > 0 {
+			params.Add(jsonutils.NewString(args.IpmiUser), "ipmi_username")
+		}
 		if len(args.IpmiPasswd) > 0 {
 			params.Add(jsonutils.NewString(args.IpmiPasswd), "ipmi_password")
 		}
@@ -505,20 +513,15 @@ func init() {
 		return nil
 	})
 
-	type HostSetSchedtagOptions struct {
-		ID       string   `help:"ID or Name of host"`
-		Schedtag []string `help:"Ids of schedtag"`
+	type HostSpecOptions struct {
+		ID string `help:"ID or name of host"`
 	}
-	R(&HostSetSchedtagOptions{}, "host-set-schedtag", "Set schedtags to a host", func(s *mcclient.ClientSession, args *HostSetSchedtagOptions) error {
-		params := jsonutils.NewDict()
-		for idx, tag := range args.Schedtag {
-			params.Add(jsonutils.NewString(tag), fmt.Sprintf("schedtag.%d", idx))
-		}
-		result, err := modules.Hosts.PerformAction(s, args.ID, "set-schedtag", params)
+	R(&HostSpecOptions{}, "host-spec", "Get host spec info", func(s *mcclient.ClientSession, args *HostSpecOptions) error {
+		spec, err := modules.Hosts.GetSpecific(s, args.ID, "spec", nil)
 		if err != nil {
 			return err
 		}
-		printObject(result)
+		printObject(spec)
 		return nil
 	})
 }

cmd/climc/shell/hostwires.go

@@ -88,11 +88,16 @@ func init() {
 	})
 
 	type HostWireDetachOptions struct {
-		HOST string `help:"ID or Name of Host"`
-		WIRE string `help:"ID or Name of Wire"`
+		HOST    string `help:"ID or Name of Host"`
+		WIRE    string `help:"ID or Name of Wire"`
+		MacAddr string `help:"Host wire mac addr"`
 	}
 	R(&HostWireDetachOptions{}, "host-wire-detach", "Detach host from wire", func(s *mcclient.ClientSession, args *HostWireDetachOptions) error {
-		result, err := modules.Hostwires.Detach(s, args.HOST, args.WIRE, nil)
+		params := jsonutils.NewDict()
+		if len(args.MacAddr) > 0 {
+			params.Set("mac_addr", jsonutils.NewString(args.MacAddr))
+		}
+		result, err := modules.Hostwires.Detach(s, args.HOST, args.WIRE, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/loadbalancerbackendgroups.go

@@ -67,4 +67,5 @@ func init() {
 		printObject(lbbackendgroup)
 		return nil
 	})
+
 }

cmd/climc/shell/loadbalancernetworks.go

@@ -15,7 +15,7 @@ func init() {
 		Network      string `help:"ID or Name of network"`
 		Ip           string `help:"search the IP address"`
 	}
-	R(&LoadbalancerNetworkListOptions{}, "loadbalancer-network-list", "List loadbalancer network pairs", func(s *mcclient.ClientSession, args *LoadbalancerNetworkListOptions) error {
+	R(&LoadbalancerNetworkListOptions{}, "lbnetwork-list", "List loadbalancer network pairs", func(s *mcclient.ClientSession, args *LoadbalancerNetworkListOptions) error {
 		var params *jsonutils.JSONDict
 		{
 			var err error
@@ -43,5 +43,4 @@ func init() {
 		printList(result, modules.Loadbalancernetworks.GetColumns(s))
 		return nil
 	})
-
 }

cmd/climc/shell/loadbalancers.go

@@ -84,4 +84,5 @@ func init() {
 		printObject(lb)
 		return nil
 	})
+
 }

cmd/climc/shell/metadata.go

@@ -19,4 +19,17 @@ func init() {
 		printList(result, []string{})
 		return nil
 	})
+
+	R(&options.TagListOptions{}, "tag-list", "List tags", func(s *mcclient.ClientSession, opts *options.TagListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Metadatas.Get(s, "tag-value-pairs", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/meteralert.go

@@ -14,32 +14,23 @@ func init() {
 	 * 创建一条报警规则
 	 */
 	type MeterAlertCreateOptions struct {
-		TYPE       string  `help:"Alert rule type" choices:"balance|resFee|monthFee"`
-		THRESHOLD  float64 `help:"Threshold value of the metric"`
-		COMPARATOR string  `help:"Comparison operator for join expressions" choices:">|<|>=|<=|=|!="`
-		RECIPIENTS string  `help:"Comma separated recipient ID"`
-		LEVEL      string  `help:"Alert level" choices:"normal|important|fatal"`
-		CHANNEL    string  `help:"Ways to send an alarm" choices:"email|mobile"`
+		Type       string  `help:"Alert rule type" required:"true" choices:"balance|resFee|monthFee"`
+		Threshold  float64 `help:"Threshold value of the metric" required:"true"`
+		Comparator string  `help:"Comparison operator for join expressions" required:"true" choices:">|<|>=|<=|=|!="`
+		Recipients string  `help:"Comma separated recipient ID" required:"true"`
+		Level      string  `help:"Alert level" required:"true" choices:"normal|important|fatal"`
+		Channel    string  `help:"Ways to send an alarm" required:"true" choices:"email|mobile"`
 		Provider   string  `help:"Name of the cloud platform"`
 		AccountId  string  `help:"ID of the cloud platform"`
+		ProjectId  string  `help:"ID of the project" required:"true"`
 	}
 	R(&MeterAlertCreateOptions{}, "meteralert-create", "Create a meter alert rule", func(s *mcclient.ClientSession, args *MeterAlertCreateOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.TYPE), "type")
-		params.Add(jsonutils.NewFloat(args.THRESHOLD), "threshold")
-		params.Add(jsonutils.NewString(args.COMPARATOR), "comparator")
-		params.Add(jsonutils.NewString(args.RECIPIENTS), "recipients")
-		params.Add(jsonutils.NewString(args.LEVEL), "level")
-		params.Add(jsonutils.NewString(args.CHANNEL), "channel")
-		if len(args.Provider) > 0 {
-			params.Add(jsonutils.NewString(args.Provider), "provider")
-		}
-		if len(args.AccountId) > 0 {
-			params.Add(jsonutils.NewString(args.AccountId), "account_id")
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
 		}
 
 		rst, err := modules.MeterAlert.Create(s, params)
-
 		if err != nil {
 			return err
 		}
@@ -89,27 +80,15 @@ func init() {
 		Type          string `help:"Alarm rule type" choices:"balance|resFee|monthFee"`
 		CloudProvider string `help:"Name of cloud provider, case sensitive"`
 		AccountId     string `help:"Cloud account ID"`
+		ProjectId     string `help:"ID of the project" required:"true"`
 		options.BaseListOptions
 	}
 	R(&MeterAlertListOptions{}, "meteralert-list", "List meter alert", func(s *mcclient.ClientSession, args *MeterAlertListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-
-			}
-			if len(args.Type) > 0 {
-				params.Add(jsonutils.NewString(args.Type), "type")
-			}
-			if len(args.CloudProvider) > 0 {
-				params.Add(jsonutils.NewString(args.CloudProvider), "provider")
-			}
-			if len(args.AccountId) > 0 {
-				params.Add(jsonutils.NewString(args.AccountId), "account_id")
-			}
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
 		}
+
 		result, err := modules.MeterAlert.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/networks.go

@@ -20,6 +20,8 @@ func init() {
 		Host       string `help:"search networks attached to a host"`
 		Vpc        string `help:"search networks belongs to a VPC"`
 		Region     string `help:"search networks belongs to a CloudRegion" json:"cloudregion"`
+		City       string `help:"search networks belongs to a city"`
+		Usable     *bool  `help:"search usable networks"`
 		ServerType string `help:"search networks belongs to a ServerType" choices:"guest|baremetal|container|pxe|ipmi"`
 	}
 	R(&NetworkListOptions{}, "network-list", "List networks", func(s *mcclient.ClientSession, opts *NetworkListOptions) error {

cmd/climc/shell/nodealert.go

@@ -0,0 +1,148 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+
+	/**
+	 * 创建一条报警规则
+	 */
+	type NodealertCreateOptions struct {
+		Type       string  `help:"Alert rule type" required:"true" choices:"guest|host"`
+		Metric     string  `help:"Metric name, include measurement and field, e.g. vm_cpu.usage_active" required:"true"`
+		NodeName   string  `help:"Name of the guest or host" required:"true"`
+		NodeID     string  `help:"ID of the guest or host" required:"true"`
+		Period     string  `help:"Specify the query time period for the data" required:"true"`
+		Window     string  `help:"Specify the query interval for the data" required:"true"`
+		Threshold  float64 `help:"Threshold value of the metric" required:"true"`
+		Comparator string  `help:"Comparison operator for join expressions" choices:">|<|>=|<=|=|!=" required:"true"`
+		Recipients string  `help:"Comma separated recipient ID" required:"true"`
+		Level      string  `help:"Alert level" required:"true" choices:"normal|important|fatal"`
+		Channel    string  `help:"Ways to send an alarm" required:"true" choices:"email|mobile|dingtalk"`
+	}
+	R(&NodealertCreateOptions{}, "nodealert-create", "Create a node alert rule", func(s *mcclient.ClientSession, args *NodealertCreateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+
+		rst, err := modules.NodeAlert.Create(s, params)
+		if err != nil {
+			return err
+		}
+
+		printObject(rst)
+		return nil
+	})
+
+	/**
+	 * 修改指定的报警规则
+	 */
+	type NodealertUpdateOptions struct {
+		ID         string  `help:"ID of the alert rule" required:"true" positional:"true"`
+		Type       string  `help:"Alert rule type" choices:"guest|host"`
+		Metric     string  `help:"Metric name, include measurement and field, such as vm_cpu.usage_active"`
+		NodeName   string  `help:"Name of the guest or host"`
+		NodeID     string  `help:"ID of the guest or host"`
+		Period     string  `help:"Specify the query time period for the data"`
+		Window     string  `help:"Specify the query interval for the data"`
+		Threshold  float64 `help:"Threshold value of the metric"`
+		Comparator string  `help:"Comparison operator for join expressions" choices:">|<|>=|<=|=|!="`
+		Recipients string  `help:"Comma separated recipient ID"`
+		Level      string  `help:"Alert level" choices:"normal|important|fatal"`
+		Channel    string  `help:"Ways to send an alarm" choices:"email|mobile"`
+	}
+	R(&NodealertUpdateOptions{}, "nodealert-update", "Update the node alert rule", func(s *mcclient.ClientSession, args *NodealertUpdateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+
+		rst, err := modules.NodeAlert.Patch(s, args.ID, params)
+
+		if err != nil {
+			return err
+		}
+
+		printObject(rst)
+		return nil
+	})
+
+	/**
+	 * 删除指定ID的报警规则
+	 */
+	type NodealertDeleteOptions struct {
+		ID string `help:"ID of node alert" required:"true" positional:"true"`
+	}
+	R(&NodealertDeleteOptions{}, "nodealert-delete", "Delete a node alert", func(s *mcclient.ClientSession, args *NodealertDeleteOptions) error {
+		alarm, err := modules.NodeAlert.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(alarm)
+		return nil
+	})
+
+	/**
+	 * 启用指定ID的报警规则状态
+	 */
+	type NodealertUpdateStatusOptions struct {
+		ID string `help:"ID of the node alert" required:"true" positional:"true"`
+	}
+	R(&NodealertUpdateStatusOptions{}, "nodealert-enable", "Enable alert rule for the specified ID", func(s *mcclient.ClientSession, args *NodealertUpdateStatusOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString("Enabled"), "status")
+
+		alarm, err := modules.NodeAlert.Patch(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(alarm)
+		return nil
+	})
+
+	/**
+	 * 禁用指定ID的报警规则状态
+	 */
+	R(&NodealertUpdateStatusOptions{}, "nodealert-disable", "Disaable alert rule for the specified ID", func(s *mcclient.ClientSession, args *NodealertUpdateStatusOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString("Disabled"), "status")
+
+		alarm, err := modules.NodeAlert.Patch(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(alarm)
+		return nil
+	})
+
+	/**
+	 * 列出报警规则
+	 */
+	type NodealertListOptions struct {
+		Type     string `help:"Alarm rule type" choices:"guest|host"`
+		Metric   string `help:"Metric name, include measurement and field, e.g. vm_cpu.usage_active"`
+		NodeName string `help:"Name of the guest or host"`
+		NodeID   string `help:"ID of the guest or host"`
+		options.BaseListOptions
+	}
+	R(&NodealertListOptions{}, "nodealert-list", "List node alert", func(s *mcclient.ClientSession, args *NodealertListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+
+		result, err := modules.NodeAlert.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.NodeAlert.GetColumns(s))
+		return nil
+	})
+}

cmd/climc/shell/routetables.go

@@ -25,10 +25,10 @@ func init() {
 				typ, _ := routeObj.GetString("type")
 				cidr, _ := routeObj.GetString("cidr")
 				next_hop_type, _ := routeObj.GetString("next_hop_type")
-				next_hop, _ := routeObj.GetString("next_hop")
+				next_hop_id, _ := routeObj.GetString("next_hop_id")
 				route := fmt.Sprintf("%8s: %18s %s", typ, cidr, next_hop_type)
-				if len(next_hop) > 0 {
-					route += fmt.Sprintf(":%s", next_hop)
+				if next_hop_id != "" {
+					route += fmt.Sprintf(":%s", next_hop_id)
 				}
 				routes = append(routes, route)
 			}

cmd/climc/shell/schedtaghosts.go

@@ -21,6 +21,7 @@ func (h *schedtagModelHelper) register() {
 		h.list(man.Slave, man.Slave.GetKeyword())
 		h.add(man, man.Slave.GetKeyword())
 		h.remove(man, man.Slave.GetKeyword())
+		h.setTags(man, man.Slave.GetKeyword())
 	}
 }
 
@@ -83,6 +84,25 @@ func (h *schedtagModelHelper) remove(man modules.JointResourceManager, kw string
 		})
 }
 
+func (h *schedtagModelHelper) setTags(man modules.JointResourceManager, kw string) {
+	R(
+		&options.SchedtagSetOptions{},
+		fmt.Sprintf("%s-set-schedtag", kw),
+		fmt.Sprintf("Set schedtags to %v", kw),
+		func(s *mcclient.ClientSession, args *options.SchedtagSetOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := man.Slave.PerformAction(s, args.ID, "set-schedtag", params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+}
+
 func init() {
 	newSchedtagModelHelper(
 		modules.Schedtaghosts,

cmd/climc/shell/schedulers.go

@@ -93,6 +93,19 @@ func init() {
 			return nil
 		})
 
+	type SchedulerCleanCacheOptions struct {
+		HostId    string `help:"ID of host" short-token:"h"`
+		SessionId string `help:"Session id" short-token:"s"`
+	}
+	R(&SchedulerCleanCacheOptions{}, "scheduler-clean-cache", "Clean scheduler hosts cache",
+		func(s *mcclient.ClientSession, args *SchedulerCleanCacheOptions) error {
+			err := modules.SchedManager.CleanCache(s, args.HostId, args.SessionId)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
 	type SchedulerHistoryListOptions struct {
 		Limit  int  `default:"50" help:"Page limit"`
 		Offset int  `default:"0" help:"Page offset"`
@@ -154,4 +167,17 @@ func init() {
 			}
 			return nil
 		})
+
+	type SyncOpt struct {
+		Wait bool `help:"wait sync finish"`
+	}
+	R(&SyncOpt{}, "scheduler-sync-sku", "Sync scheduler SKU cache",
+		func(s *mcclient.ClientSession, args *SyncOpt) error {
+			result, err := modules.SchedManager.SyncSku(s, args.Wait)
+			if err != nil {
+				return err
+			}
+			fmt.Println(result.YAMLString())
+			return nil
+		})
 }

cmd/climc/shell/servers.go

@@ -1,14 +1,19 @@
 package shell
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 
+	yaml "gopkg.in/yaml.v2"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/cmdline"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -84,6 +89,32 @@ func init() {
 		return nil
 	})
 
+	R(&options.ServerBatchMetadataOptions{}, "server-batch-add-tag", "add tags for some server", func(s *mcclient.ClientSession, opts *options.ServerBatchMetadataOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Servers.PerformClassAction(s, "batch-user-metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.ServerBatchMetadataOptions{}, "server-batch-set-tag", "Set tags for some server", func(s *mcclient.ClientSession, opts *options.ServerBatchMetadataOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Servers.PerformClassAction(s, "batch-set-user-metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.ServerMetadataOptions{}, "server-add-tag", "Set tag of a server", func(s *mcclient.ClientSession, opts *options.ServerMetadataOptions) error {
 		params, err := opts.Params()
 		if err != nil {
@@ -152,6 +183,16 @@ func init() {
 		return nil
 	})
 
+	R(&options.ServerCloneOptions{}, "server-clone", "Clone a server", func(s *mcclient.ClientSession, opts *options.ServerCloneOptions) error {
+		params := jsonutils.Marshal(opts).(*jsonutils.JSONDict)
+		res, err := modules.Servers.PerformAction(s, opts.SOURCE, "clone", params)
+		if err != nil {
+			return err
+		}
+		printObject(res)
+		return nil
+	})
+
 	R(&options.ServerLoginInfoOptions{}, "server-logininfo", "Get login info of a server", func(s *mcclient.ClientSession, opts *options.ServerLoginInfoOptions) error {
 		srvid, e := modules.Servers.GetId(s, opts.ID, nil)
 		if e != nil {
@@ -526,9 +567,16 @@ func init() {
 		params, err := options.StructToParams(opts)
 		if len(opts.Disk) > 0 {
 			params.Remove("disk.0")
+			disksConf := make([]*compute.DiskConfig, 0)
 			for i, d := range opts.Disk {
-				params.Set(fmt.Sprintf("disk.%d", i+1), jsonutils.NewString(d))
+				// params.Set(key, value)
+				diskConfig, err := cmdline.ParseDiskConfig(d, i+1)
+				if err != nil {
+					return err
+				}
+				disksConf = append(disksConf, diskConfig)
 			}
+			params.Set("disks", jsonutils.Marshal(disksConf))
 		}
 
 		if err != nil {
@@ -782,4 +830,144 @@ func init() {
 		}
 		return nil
 	})
+
+	type ServersImportFromLibvirtOptions struct {
+		CONFIG_FILE string `help:"File Path describing servers from libvirt"`
+	}
+
+	type Servers struct {
+		Mac string `yaml:"mac"`
+		Ip  string `yaml:"ip"`
+	}
+
+	type Hosts struct {
+		HostIp      string    `yaml:"host_ip"`
+		XmlFilePath string    `yaml:"xml_file_path"`
+		MonitorPath string    `yaml:"monitor_path"`
+		Servers     []Servers `yaml:"servers"`
+	}
+
+	type LibvirtImportOptions struct {
+		Hosts []Hosts `yaml:"hosts"`
+	}
+
+	R(&ServersImportFromLibvirtOptions{}, "servers-import-from-libvirt", "Import servers from libvrt", func(s *mcclient.ClientSession, args *ServersImportFromLibvirtOptions) error {
+		var (
+			rawConfig []byte
+			err       error
+		)
+
+		rawConfig, err = ioutil.ReadFile(args.CONFIG_FILE)
+		if err != nil {
+			return fmt.Errorf("Read config file %s error: %s", args.CONFIG_FILE, err)
+		}
+
+		var (
+			params []jsonutils.JSONObject
+			config = &compute.SLibvirtImportConfig{}
+		)
+
+		// Try parse as json first
+		{
+			err = json.Unmarshal(rawConfig, config)
+			if err != nil {
+				goto YAML
+			}
+			for i := 0; i < len(config.Hosts); i++ {
+				if nIp := net.ParseIP(config.Hosts[i].HostIp); nIp == nil {
+					return fmt.Errorf("Parse host ip %s failed", config.Hosts[i].HostIp)
+				}
+				for _, server := range config.Hosts[i].Servers {
+					for mac, ip := range server.MacIp {
+						if _, err := net.ParseMAC(mac); err != nil {
+							return fmt.Errorf("Parse mac %s error %s", mac, err)
+						}
+						if nIp := net.ParseIP(ip); nIp == nil {
+							return fmt.Errorf("Parse ip %s failed", ip)
+						}
+					}
+				}
+			}
+
+			goto REQUEST
+		}
+
+	YAML: // Try Parse as yaml
+		{
+			yamlConfig := &LibvirtImportOptions{}
+			err = yaml.Unmarshal(rawConfig, yamlConfig)
+			if err != nil {
+				return err
+			}
+
+			config.Hosts = make([]compute.SLibvirtHostConfig, len(yamlConfig.Hosts))
+			for i := 0; i < len(yamlConfig.Hosts); i++ {
+				if nIp := net.ParseIP(yamlConfig.Hosts[i].HostIp); nIp == nil {
+					return fmt.Errorf("Parse host ip %s failed", yamlConfig.Hosts[i].HostIp)
+				}
+				config.Hosts[i].HostIp = yamlConfig.Hosts[i].HostIp
+				config.Hosts[i].XmlFilePath = yamlConfig.Hosts[i].XmlFilePath
+				config.Hosts[i].Servers = make([]compute.SLibvirtServerConfig, len(yamlConfig.Hosts[i].Servers))
+				for j := 0; j < len(yamlConfig.Hosts[i].Servers); j++ {
+					config.Hosts[i].Servers[j].MacIp = make(map[string]string)
+					mac := yamlConfig.Hosts[i].Servers[j].Mac
+					_, err := net.ParseMAC(mac)
+					if err != nil {
+						return fmt.Errorf("Parse mac address %s error %s", mac, err)
+					}
+					ip := yamlConfig.Hosts[i].Servers[j].Ip
+					nIp := net.ParseIP(ip)
+					if len(nIp) == 0 {
+						return fmt.Errorf("Parse ip address %s failed", ip)
+					}
+					config.Hosts[i].Servers[j].MacIp[mac] = ip
+				}
+			}
+		}
+
+	REQUEST:
+		params, err = jsonutils.Marshal(config.Hosts).GetArray()
+		if err != nil {
+			return err
+		}
+		for i := 0; i < len(params); i++ {
+			val := jsonutils.NewDict()
+			val.Set(modules.Servers.KeywordPlural, params[i])
+			params[i] = val
+		}
+
+		results := modules.Servers.BatchPerformClassAction(s, "import-from-libvirt", params)
+		printBatchResults(results, modules.Servers.GetColumns(s))
+		return nil
+	})
+
+	R(&options.ServerIdOptions{}, "server-create-params", "Show server create params", func(s *mcclient.ClientSession, opts *options.ServerIdOptions) error {
+		ret, e := modules.Servers.GetSpecific(s, opts.ID, "create-params", nil)
+		if e != nil {
+			return e
+		}
+		printObject(ret)
+		return nil
+	})
+
+	type ServerExportVirtInstallCommand struct {
+		ID            string   `help:"Server Id" json:"-"`
+		LibvirtBridge string   `help:"Libvirt default bridge" json:"libvirt_bridge"`
+		ExtraCmdline  []string `help:"Extra virt-install arguments add to script, eg:'--extra-args ...', '--console ...'" json:"extra_cmdline"`
+	}
+	R(&ServerExportVirtInstallCommand{}, "server-export-virt-install-command", "Export virt-install command line from existing guest", func(s *mcclient.ClientSession, args *ServerExportVirtInstallCommand) error {
+		params := jsonutils.NewDict()
+		if len(args.LibvirtBridge) > 0 {
+			params.Set("libvirt_bridge", jsonutils.NewString(args.LibvirtBridge))
+		}
+		if len(args.ExtraCmdline) > 0 {
+			params.Set("extra_cmdline", jsonutils.NewStringArray(args.ExtraCmdline))
+		}
+		result, err := modules.Servers.GetSpecific(s, args.ID, "virt-install", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/skus.go

@@ -134,4 +134,25 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type ServerSkuSpecsListOptions struct {
+		Provider       string  `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack|Ucloud" json:"provider"`
+		PublicCloud    *bool   `help:"List objects belonging to public cloud" json:"public_cloud"`
+		Zone           string  `help:"zone Id or name"`
+		PostpaidStatus *string `help:"skus available status for postpaid instance" choices:"available|soldout"`
+		PrepaidStatus  *string `help:"skus available status for prepaid instance"  choices:"available|soldout"`
+		IngoreCache    bool    `help:"query without cache"`
+	}
+	R(&ServerSkuSpecsListOptions{}, "server-sku-specs-list", "List all avaiable Server SKU specifications", func(s *mcclient.ClientSession, args *ServerSkuSpecsListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.ServerSkus.Get(s, "instance-specs", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/snapshots.go

@@ -79,4 +79,20 @@ func init() {
 		return nil
 	})
 
+	type SnapshotCreateOptions struct {
+		Disk string `help:"Id of disk to take snapshot" json:"disk" required:"true"`
+		NAME string `help:"Name of snapshot" json:"name"`
+	}
+	R(&SnapshotCreateOptions{}, "snapshot-create", "Create a snapshot", func(s *mcclient.ClientSession, args *SnapshotCreateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		snapshot, err := modules.Snapshots.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(snapshot)
+		return nil
+	})
 }

cmd/climc/shell/specs.go

@@ -17,6 +17,7 @@ func init() {
 		HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv"`
 		Gpu      bool   `help:"Only show gpu devices"`
 		Zone     string `help:"Filter by zone id or name"`
+		Occupied bool   `help:"show occupid host" json:"-"`
 	}
 	R(&ListOptions{}, "spec", "List all kinds of model specs", func(s *mcclient.ClientSession, args *ListOptions) error {
 		var params *jsonutils.JSONDict
@@ -41,6 +42,9 @@ func init() {
 		if args.Zone != "" {
 			params.Add(jsonutils.NewString(args.Zone), "zone")
 		}
+		if args.Occupied {
+			params.Add(jsonutils.JSONFalse, "is_empty")
+		}
 		result, err := modules.Specs.GetModelSpecs(s, model, params)
 		if err != nil {
 			return err
@@ -53,10 +57,11 @@ func init() {
 		options.BaseListOptions
 		HostType string   `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv"`
 		Ncpu     int64    `help:"#CPU count of host" metavar:"<CPU_COUNT>"`
-		MemSize  string   `help:"Memory GB size"`
+		MemSize  int64    `help:"Memory MB size"`
 		DiskSpec []string `help:"Disk spec string, like 'Linux_adapter0_HDD_111Gx4'"`
 		Nic      int64    `help:"#Nics count of host" metavar:"<NIC_COUNT>"`
 		GpuModel []string `help:"GPU model, like 'GeForce GTX 1050 Ti'"`
+		Occupied bool     `help:"show occupid host" json:"-"`
 	}
 	R(&HostsQueryOptions{}, "spec-hosts-list", "List hosts according by specs", func(s *mcclient.ClientSession, args *HostsQueryOptions) error {
 		newHostSpecKeys := func() []string {
@@ -64,8 +69,11 @@ func init() {
 			if args.Ncpu > 0 {
 				keys = append(keys, fmt.Sprintf("cpu:%d", args.Ncpu))
 			}
-			if len(args.MemSize) != 0 {
-				keys = append(keys, fmt.Sprintf("mem:%s", args.MemSize))
+			if args.MemSize > 0 {
+				keys = append(keys, fmt.Sprintf("mem:%dM", args.MemSize))
+			}
+			if args.Nic > 0 {
+				keys = append(keys, fmt.Sprintf("nic:%d", args.Nic))
 			}
 			for _, gm := range args.GpuModel {
 				keys = append(keys, fmt.Sprintf("gpu_model:%s", gm))
@@ -87,6 +95,9 @@ func init() {
 		if len(args.HostType) > 0 {
 			params.Add(jsonutils.NewString(args.HostType), "host_type")
 		}
+		if args.Occupied {
+			params.Add(jsonutils.JSONFalse, "is_empty")
+		}
 		specKeys := newHostSpecKeys()
 		resp, err := modules.Specs.SpecsQueryModelObjects(s, "hosts", specKeys, params)
 		if err != nil {

cmd/climc/shell/storages.go

@@ -80,7 +80,7 @@ func init() {
 		ZONE         string `help:"Zone id of storage"`
 		Capacity     int64  `help:"Capacity of the Storage"`
 		MediumType   string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"`
-		StorageType  string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|baremetal"`
+		StorageType  string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|gpfs|baremetal"`
 		MonHost      string `help:"Ceph mon_host config"`
 		Key          string `help:"Ceph key config"`
 		Pool         string `help:"Ceph Poll Name"`
@@ -214,5 +214,4 @@ func init() {
 		printObject(storage)
 		return nil
 	})
-
 }

cmd/climc/shell/updates.go

@@ -1,6 +1,8 @@
 package shell
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -27,21 +29,29 @@ func init() {
 	})
 
 	type SUpdatePerformOptions struct {
-		Cmp bool `help:"update all the compute nodes automatically"`
+		Cmp     bool `help:"update Controller And all the Compute nodes automatically"`
+		CmpOnly bool `help:"Updates only computes nodes, excluding controller nodes"`
 	}
 
 	R(&SUpdatePerformOptions{}, "update-perform", "Update the Controler", func(s *mcclient.ClientSession, args *SUpdatePerformOptions) error {
 		params := jsonutils.NewDict()
-		if args.Cmp {
-			params.Add(jsonutils.JSONTrue, "cmp")
+
+		if args.Cmp && args.CmpOnly {
+			return fmt.Errorf("--cmp and --cmp-only can't go together")
 		}
 
-		result, err := modules.Updates.List(s, nil)
+		if args.Cmp {
+			params.Add(jsonutils.JSONTrue, "cmp")
+		} else if args.CmpOnly {
+			params.Add(jsonutils.JSONTrue, "cmp_only")
+		}
 
+		modules.Updates.PerformAction(s, "", "", params)
+
+		result, err := modules.Updates.List(s, nil)
 		if err != nil {
 			return err
 		}
-		modules.Updates.DoUpdate(s, params)
 		printList(result, modules.Updates.GetColumns(s))
 		return nil
 	})

cmd/climc/shell/webconsole.go

@@ -5,17 +5,23 @@ import (
 	"net/url"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	o "yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/webconsole/command"
+	"yunion.io/x/onecloud/pkg/webconsole/session"
+)
+
+const (
+	DefaultWebconsoleServer = "https://console.yunion.cn/web-console"
 )
 
 func init() {
 	handleResult := func(opt o.WebConsoleOptions, obj jsonutils.JSONObject) error {
 		if opt.WebconsoleUrl == "" {
-			printObject(obj)
-			return nil
+			opt.WebconsoleUrl = DefaultWebconsoleServer
 		}
 		u, err := url.Parse(opt.WebconsoleUrl)
 		if err != nil {
@@ -25,6 +31,25 @@ func init() {
 		if err != nil {
 			return err
 		}
+		var query url.Values
+		connUrl, err := url.ParseRequestURI(connParams)
+		if err == nil {
+			query = connUrl.Query()
+		} else {
+			query, err = url.ParseQuery(connParams)
+			if err != nil {
+				return err
+			}
+		}
+		protocol := query.Get("protocol")
+		if !utils.IsInStringArray(protocol, []string{
+			command.PROTOCOL_TTY, session.VNC,
+			session.SPICE, session.WMKS,
+		}) {
+			fmt.Println(connParams)
+			return nil
+		}
+
 		u.RawQuery = connParams
 		fmt.Println(u.String())
 		return nil
@@ -66,7 +91,11 @@ func init() {
 	})
 
 	R(&o.WebConsoleSshOptions{}, "webconsole-ssh", "Connect ssh webconsole", func(s *mcclient.ClientSession, args *o.WebConsoleSshOptions) error {
-		ret, err := modules.WebConsole.DoSshConnect(s, args.IP, nil)
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := modules.WebConsole.DoSshConnect(s, args.IP, params)
 		if err != nil {
 			return err
 		}

cmd/cloudir/main.go

@@ -1,7 +1,12 @@
 package main
 
-import "yunion.io/x/onecloud/pkg/cloudir/service"
+import (
+	"yunion.io/x/onecloud/pkg/cloudir/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/cloutpost/main.go

@@ -1,7 +1,12 @@
 package main
 
-import "yunion.io/x/onecloud/pkg/cloutpost/service"
+import (
+	"yunion.io/x/onecloud/pkg/cloutpost/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/esxicli/main.go

@@ -74,7 +74,7 @@ func newClient(options *BaseOptions) (*esxi.SESXiClient, error) {
 		return nil, fmt.Errorf("Missing password")
 	}
 
-	return esxi.NewESXiClient("", "", options.Host, options.Port, options.Account, options.Password)
+	return esxi.NewESXiClient2("", "", options.Host, options.Port, options.Account, options.Password, false)
 }
 
 func main() {

cmd/glance/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/image/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/host-image/main.go

@@ -1,7 +1,12 @@
 package main
 
-import "yunion.io/x/onecloud/pkg/hostimage"
+import (
+	"yunion.io/x/onecloud/pkg/hostimage"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
+	defer atexit.Handle()
+
 	hostimage.StartService()
 }

cmd/host/main.go

@@ -1,8 +1,17 @@
 package main
 
-import "yunion.io/x/onecloud/pkg/hostman"
+import (
+	"yunion.io/x/onecloud/pkg/cloudcommon/service"
+	"yunion.io/x/onecloud/pkg/hostman"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
 
 func main() {
-	var srv = hostman.SHostService{}
+	defer atexit.Handle()
+
+	var srv = &hostman.SHostService{}
+	srv.SServiceBase = &service.SServiceBase{
+		Service: srv,
+	}
 	srv.StartService()
 }

cmd/lbagent/main.go

@@ -12,9 +12,12 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/lbagent"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	opts := &lbagent.Options{}
 	commonOpts := &opts.CommonOptions
 	{

cmd/logger/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/logger/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/openstackcli/main.go

@@ -20,6 +20,7 @@ type BaseOptions struct {
 	Password     string `help:"Password" default:"$OPENSTACK_PASSWORD"`
 	Project      string `help:"Project" default:"$OPENSTACK_PROJECT"`
 	EndpointType string `help:"Project" default:"$OPENSTACK_ENDPOINT_TYPE|internal"`
+	DomainName   string `help:"DomainName" default:"$OPENSTACK_DOMAIN_NAME"`
 	RegionID     string `help:"RegionId" default:"$OPENSTACK_REGION_ID"`
 	SUBCOMMAND   string `help:"openstackcli subcommand" subcommand:"true"`
 }
@@ -77,7 +78,7 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
-	cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.Debug)
+	cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.DomainName, options.Debug)
 	if err != nil {
 		return nil, err
 	}

cmd/region-dns/main.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/mholt/caddy/startupshutdown"
 
 	_ "yunion.io/x/onecloud/pkg/dns"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 var directives = []string{
@@ -44,5 +45,7 @@ func init() {
 }
 
 func main() {
+	defer atexit.Handle()
+
 	coremain.Run()
 }

cmd/region/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"yunion.io/x/onecloud/pkg/compute/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/scheduler/main.go

@@ -1,17 +1,20 @@
 package main
 
 import (
-	"os"
-
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/scheduler/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
 func main() {
+	exitCode := 0
+	defer func() {
+		atexit.Exit(exitCode)
+	}()
+
 	if err := service.StartService(); err != nil {
 		log.Errorln(err)
-		os.Exit(-1)
+		exitCode = -1
 	}
-	os.Exit(0)
 }

cmd/torrent/main.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/nodeid"
 	"yunion.io/x/onecloud/pkg/util/torrentutils"
@@ -48,6 +49,8 @@ func exitSignalHandlers(client *torrent.Client) {
 }
 
 func main() {
+	defer atexit.Handle()
+
 	options := Options{}
 
 	parser, err := structarg.NewArgumentParser(&options, "torrent-srv", "bit-torrent server", "2018")

cmd/webconsole/main.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

cmd/yunionconf/main.go

@@ -6,9 +6,12 @@ Yunion Conf Service
 */
 
 import (
+	"yunion.io/x/onecloud/pkg/util/atexit"
 	"yunion.io/x/onecloud/pkg/yunionconf/service"
 )
 
 func main() {
+	defer atexit.Handle()
+
 	service.StartService()
 }

pkg/apis/billing/billing_const.go

@@ -0,0 +1,6 @@
+package billing
+
+const (
+	BILLING_TYPE_POSTPAID = "postpaid"
+	BILLING_TYPE_PREPAID  = "prepaid"
+)

pkg/apis/billing/doc.go

@@ -0,0 +1 @@
+package billing // import "yunion.io/x/onecloud/pkg/apis/billing"

pkg/apis/compute/agent_const.go

@@ -0,0 +1,15 @@
+package compute
+
+const (
+	BAREMETAL_AGENT_ENABLED  = "enabled"
+	BAREMETAL_AGENT_DISABLED = "disabled"
+	BAREMETAL_AGENT_OFFLINE  = "offline"
+)
+
+type TAgentType string
+
+const (
+	AgentTypeBaremetal = TAgentType("baremetal")
+	AgentTypeEsxi      = TAgentType("esxiagent")
+	AgentTypeDefault   = AgentTypeBaremetal
+)

pkg/apis/compute/api.go

@@ -11,6 +11,7 @@ type SchedtagConfig struct {
 
 	Id       string `json:"id"`
 	Strategy string `json:"strategy"`
+	Weight   int    `json:"weight"`
 }
 
 type NetworkConfig struct {
@@ -59,7 +60,7 @@ type DiskConfig struct {
 	Medium          string            `json:"medium"`
 	ImageProperties map[string]string `json:"image_properties"`
 
-	Storage string `json:"storage"`
+	Storage string `json:"storage_id"`
 	DiskId  string `json:"disk_id"`
 }
 
@@ -162,6 +163,24 @@ type ServerCreateInput struct {
 
 	// DEPRECATE or not used fields
 	Baremetal bool `json:"baremetal"`
+
+	// Used to store BaremetalConvertHypervisorTaskId
+	ParentTaskId string `json:"__parent_task_id,omitempty"`
+	// default stroage type if host is given
+	DefaultStorageType string `json:"default_storage_type,omitempty"`
+}
+
+type ServerCloneInput struct {
+	apis.Meta
+
+	Name      string `json:"name"`
+	AutoStart bool   `json:"auto_start"`
+
+	EipBw         int    `json:"eip_bw,omitzero"`
+	EipChargeType string `json:"eip_charge_type,omitempty"`
+	Eip           string `json:"eip,omitempty"`
+
+	PreferHost string `json:"prefer_host_id"`
 }
 
 type ServerDeployInput struct {
@@ -170,7 +189,7 @@ type ServerDeployInput struct {
 	Id string
 
 	Keypair       string          `json:"keypair"`
-	DeleteKeypair bool            `json:"__delete_keypair__"`
+	DeleteKeypair *bool           `json:"__delete_keypair__"`
 	DeployConfigs []*DeployConfig `json:"deploy_configs"`
 	ResetPassword *bool           `json:"reset_password"`
 	Password      string          `json:"password"`

pkg/apis/compute/cachedimages_const.go

@@ -0,0 +1,17 @@
+package compute
+
+const (
+	CACHED_IMAGE_STATUS_INIT         = "init"
+	CACHED_IMAGE_STATUS_SAVING       = "saving"
+	CACHED_IMAGE_STATUS_CACHING      = "caching"
+	CACHED_IMAGE_STATUS_READY        = "ready"
+	CACHED_IMAGE_STATUS_DELETING     = "deleting"
+	CACHED_IMAGE_STATUS_CACHE_FAILED = "cache_fail"
+
+	DOWNLOAD_SESSION_LENGTH = 3600 * 3 // 3 hour
+)
+
+const (
+	CACHED_IMAGE_REFRESH_SECONDS                  = 900   // 15 minutes
+	CACHED_IMAGE_REFERENCE_SESSION_EXPIRE_SECONDS = 86400 // 1 day
+)

pkg/apis/compute/cloudaccount_const.go

@@ -0,0 +1,57 @@
+package compute
+
+const (
+	CLOUD_PROVIDER_INIT          = "init"
+	CLOUD_PROVIDER_CONNECTED     = "connected"
+	CLOUD_PROVIDER_DISCONNECTED  = "disconnected"
+	CLOUD_PROVIDER_START_DELETE  = "start_delete"
+	CLOUD_PROVIDER_DELETING      = "deleting"
+	CLOUD_PROVIDER_DELETED       = "deleted"
+	CLOUD_PROVIDER_DELETE_FAILED = "delete_failed"
+
+	CLOUD_PROVIDER_SYNC_STATUS_QUEUED  = "queued"
+	CLOUD_PROVIDER_SYNC_STATUS_SYNCING = "syncing"
+	CLOUD_PROVIDER_SYNC_STATUS_IDLE    = "idle"
+
+	CLOUD_PROVIDER_ONECLOUD  = "OneCloud"
+	CLOUD_PROVIDER_VMWARE    = "VMware"
+	CLOUD_PROVIDER_ALIYUN    = "Aliyun"
+	CLOUD_PROVIDER_QCLOUD    = "Qcloud"
+	CLOUD_PROVIDER_AZURE     = "Azure"
+	CLOUD_PROVIDER_AWS       = "Aws"
+	CLOUD_PROVIDER_HUAWEI    = "Huawei"
+	CLOUD_PROVIDER_OPENSTACK = "OpenStack"
+	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
+
+	CLOUD_PROVIDER_HEALTH_NORMAL        = "normal"        // 远端处于健康状态
+	CLOUD_PROVIDER_HEALTH_INSUFFICIENT  = "insufficient"  // 不足按需资源余额
+	CLOUD_PROVIDER_HEALTH_SUSPENDED     = "suspended"     // 远端处于冻结状态
+	CLOUD_PROVIDER_HEALTH_ARREARS       = "arrears"       // 远端处于欠费状态
+	CLOUD_PROVIDER_HEALTH_UNKNOWN       = "unknown"       // 未知状态，查询失败
+	CLOUD_PROVIDER_HEALTH_NO_PERMISSION = "no permission" // 没有权限获取账单信息
+)
+
+var (
+	CLOUD_PROVIDER_VALID_STATUS        = []string{CLOUD_PROVIDER_CONNECTED}
+	CLOUD_PROVIDER_VALID_HEALTH_STATUS = []string{CLOUD_PROVIDER_HEALTH_NORMAL, CLOUD_PROVIDER_HEALTH_NO_PERMISSION}
+
+	CLOUD_PROVIDERS = []string{
+		CLOUD_PROVIDER_ONECLOUD,
+		CLOUD_PROVIDER_VMWARE,
+		CLOUD_PROVIDER_ALIYUN,
+		CLOUD_PROVIDER_QCLOUD,
+		CLOUD_PROVIDER_AZURE,
+		CLOUD_PROVIDER_AWS,
+		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_OPENSTACK,
+		CLOUD_PROVIDER_UCLOUD,
+	}
+)
+
+const (
+	CLOUD_ENV_PUBLIC_CLOUD  = "public"
+	CLOUD_ENV_PRIVATE_CLOUD = "private"
+	CLOUD_ENV_ON_PREMISE    = "onpremise"
+
+	CLOUD_ENV_PRIVATE_ON_PREMISE = "private_or_onpremise"
+)

pkg/apis/compute/cloudregions_const.go

@@ -0,0 +1,142 @@
+package compute
+
+const (
+	CLOUD_REGION_STATUS_INSERVER     = "inservice"
+	CLOUD_REGION_STATUS_OUTOFSERVICE = "outofservice"
+
+	DEFAULT_REGION_ID = "default"
+
+	// 中国
+	CITY_QING_DAO      = "Qingdao"     //青岛
+	CITY_BEI_JING      = "Beijing"     //北京
+	CITY_ZHANG_JIA_KOU = "Zhangjiakou" //张家口
+	CITY_HU_HE_HAO_TE  = "Huhehaote"   //呼和浩特
+	CITY_HANG_ZHOU     = "Hangzhou"    //杭州
+	CITY_SHANG_HAI     = "Shanghai"    //上海
+	CITY_SHEN_ZHEN     = "Shenzhen"    //深圳
+	CITY_HONG_KONG     = "Hongkong"    //香港
+	CITY_NING_XIA      = "Ningxia"     //宁夏
+	CITY_GUANG_ZHOU    = "Guangzhou"   //广州
+	CITY_GUI_YANG      = "Guiyang"     //贵阳
+	CITY_TAIPEI        = "Taipei"      //台北市
+	CITY_KAOHSIUNG     = "Kaohsiung"   //高雄市
+	CITY_CHENG_DU      = "Chengdu"     //成都
+	CITY_CHONG_QING    = "Chongqing"   //重庆
+
+	// 日本
+	CITY_TOKYO = "Tokyo" //东京
+	CITY_OSAKA = "Osaka" //大阪市
+
+	// 新加坡
+	CITY_SINGAPORE = "Singapore" //新加坡
+
+	// 澳大利亚
+	CITY_SYDNEY     = "Sydney"     //悉尼
+	CITY_YARRALUMLA = "Yarralumla" //亚拉伦拉
+	CITY_MELBOURNE  = "Melbourne"  //墨尔本
+
+	// 马来西亚
+	CITY_KUALA_LUMPUR = "Kuala Lumpur" //吉隆坡
+
+	// 印度尼西亚
+	CITY_JAKARTA = "Jakarta" //雅加达
+
+	// 印度
+	CITY_MUMBAI      = "Mumbai"      //孟买
+	CITY_KANCHIPURAM = "Kanchipuram" //甘吉布勒姆
+	CITY_MAHARASHTRA = "Maharashtra" //马哈拉施特拉邦
+
+	// 美国
+	CITY_VIRGINIA      = "Virginia"      //弗吉尼亚
+	CITY_SILICONVALLEY = "Siliconvalley" //硅谷
+	CITY_OHIO          = "Ohio"          //俄亥俄州
+	CITY_N_VIRGINIA    = "N. Virginia"   //北弗吉尼亚
+	CITY_N_CALIFORNIA  = "N. California" //北加州
+	CITY_OREGON        = "Oregon"        //俄勒冈州
+	CITY_LOS_ANGELES   = "Los Angeles"   //洛杉矶
+	CITY_SAN_FRANCISCO = "San Francisco" //旧金山
+	CITY_UTAH          = "Utah"          //犹他州
+	CITY_WASHINGTON    = "Washington"    //华盛顿
+	CITY_TEXAS         = "Texas"         //德克萨斯
+	CITY_CHICAGO       = "Chicago"       //芝加哥
+	CITY_IOWA          = "Iowa"          //爱荷华
+	CITY_US_GOV_WEST   = "us-gov-west"   //???
+
+	// 英国
+	CITY_LONDON      = "London"      //伦敦
+	CITY_HALTON      = "Halton"      //哈尔顿
+	CITY_WEST_SUSSEX = "West Sussex" //西苏塞克斯
+
+	// 阿拉伯联合酋长国
+	CITY_DUBAI = "Dubai" //迪拜
+
+	// 德国
+	CITY_FRANKFURT = "Frankfurt" //法兰克福
+
+	// 韩国
+	CITY_SEOUL = "Seoul" //首尔
+	CITY_BUSAN = "Busan" //釜山
+
+	// 加拿大
+	CITY_CANADA_CENTRAL = "Canada Central" //加拿大中部
+	CITY_QUEBEC         = "Quebec"         //魁北克市
+	CITY_TORONTO        = "Toronto"        //多伦多
+
+	// 爱尔兰
+	CITY_IRELAND = "Ireland" //爱尔兰
+	CITY_DUBLIN  = "Dublin"  //都柏林
+
+	// 法国
+	CITY_PARIS  = "Paris"  //巴黎
+	CITY_ALLIER = "Allier" //阿利埃河
+	CITY_TARN   = "Tarn"   //塔恩
+
+	// 瑞典
+	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
+
+	// 巴西
+	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+
+	// 荷兰
+	CITY_HOLLAND = "Holland" //荷兰
+
+	// 南非
+	CITY_PRETORIA  = "Pretoria"  //比勒陀利亚
+	CITY_CAPE_TOWN = "Cape Town" //开普敦
+
+	// 泰国
+	CITY_BANGKOK = "Bangkok" //曼谷
+
+	// 俄罗斯
+	CITY_MOSCOW = "Moscow" //莫斯科
+
+	// 尼日利亚
+	CITY_LAGOS = "Lagos" //拉哥斯
+
+	// 越南
+	CITY_HO_CHI_MINH = "Ho Chi Minh" //???
+
+	COUNTRY_CODE_CN = "CN" //中国
+	COUNTRY_CODE_JP = "JP" //日本
+	COUNTRY_CODE_SG = "SG" //新加坡
+	COUNTRY_CODE_AU = "AU" //澳大利亚
+	COUNTRY_CODE_MY = "MY" //马来西亚
+	COUNTRY_CODE_ID = "ID" //印度尼西亚
+	COUNTRY_CODE_IN = "IN" //印度
+	COUNTRY_CODE_US = "US" //美国
+	COUNTRY_CODE_GB = "GB" //英国
+	COUNTRY_CODE_AE = "AE" //阿拉伯联合酋长国
+	COUNTRY_CODE_DE = "DE" //德国
+	COUNTRY_CODE_KR = "KR" //韩国
+	COUNTRY_CODE_CA = "CA" //加拿大
+	COUNTRY_CODE_IE = "IE" //爱尔兰
+	COUNTRY_CODE_FR = "FR" //法国
+	COUNTRY_CODE_SE = "SE" //瑞典
+	COUNTRY_CODE_BR = "BR" //巴西
+	COUNTRY_CODE_NL = "NL" //荷兰
+	COUNTRY_CODE_ZA = "ZA" //南非
+	COUNTRY_CODE_TH = "TH" //泰国
+	COUNTRY_CODE_RU = "RU" //俄罗斯
+	COUNTRY_CODE_NG = "NG" //尼日利亚
+	COUNTRY_CODE_VN = "VN" //越南
+)

pkg/apis/compute/disk_const.go

@@ -37,4 +37,6 @@ const (
 	DISK_TYPE_VOLUME = "volume"
 
 	DISK_BACKING_IMAGE = "image"
+
+	DISK_SIZE_AUTOEXTEND = -1
 )

pkg/apis/compute/elasticips_const.go

@@ -0,0 +1,25 @@
+package compute
+
+const (
+	EIP_MODE_INSTANCE_PUBLICIP = "public_ip"
+	EIP_MODE_STANDALONE_EIP    = "elastic_ip"
+
+	EIP_ASSOCIATE_TYPE_SERVER = "server"
+
+	EIP_STATUS_READY           = "ready"
+	EIP_STATUS_UNKNOWN         = "unknown"
+	EIP_STATUS_ALLOCATE        = "allocate"
+	EIP_STATUS_ALLOCATE_FAIL   = "allocate_fail"
+	EIP_STATUS_DEALLOCATE      = "deallocate"
+	EIP_STATUS_DEALLOCATE_FAIL = "deallocate_fail"
+	EIP_STATUS_ASSOCIATE       = "associate"
+	EIP_STATUS_ASSOCIATE_FAIL  = "associate_fail"
+	EIP_STATUS_DISSOCIATE      = "dissociate"
+	EIP_STATUS_DISSOCIATE_FAIL = "dissociate_fail"
+
+	EIP_STATUS_CHANGE_BANDWIDTH = "change_bandwidth"
+
+	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
+	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
+	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
+)

pkg/apis/compute/esxi_const.go

@@ -0,0 +1,5 @@
+package compute
+
+const (
+	ESXI_IMAGE_CACHE_TMP_PREFIX = "yunioncloud.imagecache"
+)

pkg/apis/compute/guest_const.go

@@ -22,6 +22,8 @@ const (
 	VM_START_STOP      = "start_stop"
 	VM_STOPPING        = "stopping"
 	VM_STOP_FAILED     = "stop_fail" // # = running
+	VM_RENEWING        = "renewing"
+	VM_RENEW_FAILED    = "renew_failed"
 
 	VM_BACKUP_STARTING         = "backup_starting"
 	VM_BACKUP_CREATING         = "backup_creating"
@@ -56,13 +58,13 @@ const (
 	VM_REBUILD_ROOT       = "rebuild_root"
 	VM_REBUILD_ROOT_FAIL  = "rebuild_root_fail"
 
-	VM_START_SNAPSHOT  = "snapshot_start"
-	VM_SNAPSHOT        = "snapshot"
-	VM_SNAPSHOT_DELETE = "snapshot_delete"
-	VM_BLOCK_STREAM    = "block_stream"
-	VM_MIRROR_FAIL     = "mirror_failed"
-	VM_SNAPSHOT_SUCC   = "snapshot_succ"
-	VM_SNAPSHOT_FAILED = "snapshot_failed"
+	VM_START_SNAPSHOT    = "snapshot_start"
+	VM_SNAPSHOT          = "snapshot"
+	VM_SNAPSHOT_DELETE   = "snapshot_delete"
+	VM_BLOCK_STREAM      = "block_stream"
+	VM_BLOCK_STREAM_FAIL = "block_stream_fail"
+	VM_SNAPSHOT_SUCC     = "snapshot_succ"
+	VM_SNAPSHOT_FAILED   = "snapshot_failed"
 
 	VM_SYNCING_STATUS = "syncing"
 	VM_SYNC_CONFIG    = "sync_config"
@@ -88,6 +90,9 @@ const (
 
 	VM_ADMIN = "admin"
 
+	VM_IMPORT        = "import"
+	VM_IMPORT_FAILED = "import_fail"
+
 	SHUTDOWN_STOP      = "stop"
 	SHUTDOWN_TERMINATE = "terminate"
 
@@ -165,3 +170,18 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 	HOST_TYPE_OPENSTACK:  HYPERVISOR_OPENSTACK,
 	HOST_TYPE_UCLOUD:     HYPERVISOR_UCLOUD,
 }
+
+const (
+	VM_AWS_DEFAULT_LOGIN_USER   = "ec2user"
+	VM_AZURE_DEFAULT_LOGIN_USER = "toor"
+
+	VM_METADATA_APP_TAGS            = "app_tags"
+	VM_METADATA_CREATE_PARAMS       = "create_params"
+	VM_METADATA_LOGIN_ACCOUNT       = "login_account"
+	VM_METADATA_LOGIN_KEY           = "login_key"
+	VM_METADATA_LOGIN_KEY_TIMESTAMP = "login_key_timestamp"
+	VM_METADATA_OS_ARCH             = "os_arch"
+	VM_METADATA_OS_DISTRO           = "os_distribution"
+	VM_METADATA_OS_NAME             = "os_name"
+	VM_METADATA_OS_VERSION          = "os_version"
+)

pkg/apis/compute/guest_import_api.go

@@ -0,0 +1,72 @@
+package compute
+
+type SImportNic struct {
+	Index     int    `json:"index"`
+	Bridge    string `json:"bridge"`
+	Domain    string `json:"domain"`
+	Ip        string `json:"ip"`
+	Vlan      int    `json:"vlan"`
+	Driver    string `json:"driver"`
+	Masklen   int    `json:"masklen"`
+	Virtual   bool   `json:"virtual"`
+	Manual    bool   `json:"manual"`
+	WireId    string `json:"wire_id"`
+	NetId     string `json:"net_id"`
+	Mac       string `json:"mac"`
+	BandWidth int    `json:"bw"`
+	Dns       string `json:"dns"`
+	Net       string `json:"net"`
+	Interface string `json:"interface"`
+	Gateway   string `json:"gateway"`
+	Ifname    string `json:"ifname"`
+}
+
+type SImportDisk struct {
+	Index      int    `json:"index"`
+	DiskId     string `json:"disk_id"`
+	Driver     string `json:"driver"`
+	CacheMode  string `json:"cache_mode"`
+	AioMode    string `json:"aio_mode"`
+	SizeMb     int    `json:"size"`
+	Format     string `json:"format"`
+	Fs         string `json:"fs"`
+	Mountpoint string `json:"mountpoint"`
+	Dev        string `json:"dev"`
+	TemplateId string `json:"template_id"`
+	AccessPath string `json:"AccessPath"`
+	Backend    string `json:"Backend"`
+}
+
+type SImportGuestDesc struct {
+	Id          string            `json:"uuid"`
+	Name        string            `json:"name"`
+	Nics        []SImportNic      `json:"nics"`
+	Disks       []SImportDisk     `json:"disks"`
+	Metadata    map[string]string `json:"metadata"`
+	MemSizeMb   int               `json:"mem"`
+	Cpu         int               `json:"cpu"`
+	TemplateId  string            `json:"template_id"`
+	ImagePath   string            `json:"image_path"`
+	Vdi         string            `json:"vdi"`
+	Hypervisor  string            `json:"hypervisor"`
+	HostId      string            `json:"host"`
+	BootOrder   string            `json:"boot_order"`
+	IsSystem    bool              `json:"is_system"`
+	Description string            `json:"description"`
+	MonitorPath string            `json:"monitor_path"`
+}
+
+type SLibvirtServerConfig struct {
+	MacIp map[string]string `json:"mac_ip"`
+}
+
+type SLibvirtHostConfig struct {
+	Servers     []SLibvirtServerConfig `json:"servers"`
+	XmlFilePath string                 `json:"xml_file_path"`
+	MonitorPath string                 `json:"monitor_path"`
+	HostIp      string                 `json:"host_ip"`
+}
+
+type SLibvirtImportConfig struct {
+	Hosts []SLibvirtHostConfig `json:"hosts"`
+}

pkg/apis/compute/host.go

@@ -0,0 +1,45 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type HostSpec struct {
+	apis.Meta
+
+	Cpu         int            `json:"cpu"`
+	Mem         int            `json:"mem"`
+	NicCount    int            `json:"nic_count"`
+	Manufacture string         `json:"manufacture"`
+	Model       string         `json:"model"`
+	Disk        DiskDriverSpec `json:"disk"`
+	Driver      string         `json:"driver"`
+}
+
+type DiskDriverSpec map[string]DiskAdapterSpec
+
+type DiskAdapterSpec map[string][]*DiskSpec
+
+type DiskSpec struct {
+	apis.Meta
+
+	Type       string `json:"type"`
+	Size       int64  `json:"size"`
+	StartIndex int    `json:"start_index"`
+	EndIndex   int    `json:"end_index"`
+	Count      int    `json:"count"`
+}

pkg/apis/compute/loadbalancer_const.go

@@ -66,6 +66,31 @@ var LB_BACKENDGROUP_TYPE = choices.NewChoices(
 	LB_BACKENDGROUP_TYPE_MASTER_SLAVE,
 )
 
+const (
+	LB_ALIYUN_SPEC_SHAREABLE = "" //性能共享型
+	LB_ALIYUN_SPEC_S1_SMALL  = "slb.s1.small"
+	LB_ALIYUN_SPEC_S2_SMALL  = "slb.s2.small"
+	LB_ALIYUN_SPEC_S3_SMALL  = "slb.s3.small"
+	LB_ALIYUN_SPEC_S2_MEDIUM = "slb.s2.medium"
+	LB_ALIYUN_SPEC_S3_MEDIUM = "slb.s3.medium"
+	LB_ALIYUN_SPEC_S3_LARGE  = "slb.s3.large"
+)
+
+const (
+	LB_MbpsMin = 0
+	LB_MbpsMax = 10000
+)
+
+var LB_ALIYUN_SPECS = choices.NewChoices(
+	LB_ALIYUN_SPEC_SHAREABLE,
+	LB_ALIYUN_SPEC_S1_SMALL,
+	LB_ALIYUN_SPEC_S2_SMALL,
+	LB_ALIYUN_SPEC_S3_SMALL,
+	LB_ALIYUN_SPEC_S2_MEDIUM,
+	LB_ALIYUN_SPEC_S3_MEDIUM,
+	LB_ALIYUN_SPEC_S3_LARGE,
+)
+
 // Load Balancer network type (vpc or classic) determines viable backend
 // servers (they should be from the same network type as the load balancer).
 //
@@ -217,6 +242,7 @@ const (
 	LB_SCHEDULER_WLC = "wlc" // weighted least connection
 	LB_SCHEDULER_SCH = "sch" // source-ip-based consistent hash
 	LB_SCHEDULER_TCH = "tch" // 4-tuple-based consistent hash
+	LB_SCHEDULER_QCH = "qch"
 )
 
 var LB_SCHEDULER_TYPES = choices.NewChoices(
@@ -227,15 +253,32 @@ var LB_SCHEDULER_TYPES = choices.NewChoices(
 	LB_SCHEDULER_TCH,
 )
 
+var LB_ALIYUN_UDP_SCHEDULER_TYPES = choices.NewChoices(
+	LB_SCHEDULER_RR,
+	LB_SCHEDULER_WRR,
+	LB_SCHEDULER_WLC,
+	LB_SCHEDULER_SCH,
+	LB_SCHEDULER_TCH,
+	LB_SCHEDULER_QCH,
+)
+
+var LB_ALIYUN_COMMON_SCHEDULER_TYPES = choices.NewChoices(
+	LB_SCHEDULER_RR,
+	LB_SCHEDULER_WRR,
+	LB_SCHEDULER_WLC,
+)
+
 // TODO raw type
 const (
 	LB_BACKEND_GUEST = "guest"
 	LB_BACKEND_HOST  = "host"
+	LB_BACKEND_IP    = "ip"
 )
 
 var LB_BACKEND_TYPES = choices.NewChoices(
 	LB_BACKEND_GUEST,
 	LB_BACKEND_HOST,
+	LB_BACKEND_IP,
 )
 
 const (
@@ -256,6 +299,12 @@ const (
 	LB_CHARGE_TYPE_BY_HOUR      = "hour"
 )
 
+var LB_CHARGE_TYPES = choices.NewChoices(
+	LB_CHARGE_TYPE_BY_TRAFFIC,
+	LB_CHARGE_TYPE_BY_BANDWIDTH,
+	LB_CHARGE_TYPE_BY_HOUR,
+)
+
 const (
 	LB_HA_STATE_MASTER  = "MASTER"
 	LB_HA_STATE_BACKUP  = "BACKUP"

pkg/apis/compute/network_const.go

@@ -35,4 +35,9 @@ var (
 		NETWORK_TYPE_PXE,
 		NETWORK_TYPE_IPMI,
 	}
+
+	REGINAL_NETWORK_PROVIDERS = []string{
+		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_UCLOUD,
+	}
 )

pkg/apis/compute/sku_const.go

@@ -0,0 +1,28 @@
+package compute
+
+const (
+	SkuCategoryGeneralPurpose      = "general_purpose"      // 通用型
+	SkuCategoryBurstable           = "burstable"            // 突发性能型
+	SkuCategoryComputeOptimized    = "compute_optimized"    // 计算优化型
+	SkuCategoryMemoryOptimized     = "memory_optimized"     // 内存优化型
+	SkuCategoryStorageIOOptimized  = "storage_optimized"    // 存储IO优化型
+	SkuCategoryHardwareAccelerated = "hardware_accelerated" // 硬件加速型
+	SkuCategoryHighStorage         = "high_storage"         // 高存储型
+	SkuCategoryHighMemory          = "high_memory"          // 高内存型
+)
+
+const (
+	SkuStatusAvailable = "available"
+	SkuStatusSoldout   = "soldout"
+)
+
+var InstanceFamilies = map[string]string{
+	SkuCategoryGeneralPurpose:      "g1",
+	SkuCategoryBurstable:           "t1",
+	SkuCategoryComputeOptimized:    "c1",
+	SkuCategoryMemoryOptimized:     "r1",
+	SkuCategoryStorageIOOptimized:  "i1",
+	SkuCategoryHardwareAccelerated: "",
+	SkuCategoryHighStorage:         "hc1",
+	SkuCategoryHighMemory:          "hr1",
+}

pkg/apis/compute/snapshot_const.go

@@ -0,0 +1,14 @@
+package compute
+
+const (
+	// create by
+	SNAPSHOT_MANUAL = "manual"
+	SNAPSHOT_AUTO   = "auto"
+
+	SNAPSHOT_CREATING    = "creating"
+	SNAPSHOT_ROLLBACKING = "rollbacking"
+	SNAPSHOT_FAILED      = "create_failed"
+	SNAPSHOT_READY       = "ready"
+	SNAPSHOT_DELETING    = "deleting"
+	SNAPSHOT_UNKNOWN     = "unknown"
+)

pkg/apis/compute/storage_const.go

@@ -9,6 +9,7 @@ const (
 	STORAGE_NAS       = "nas"
 	STORAGE_VSAN      = "vsan"
 	STORAGE_NFS       = "nfs"
+	STORAGE_GPFS      = "gpfs"
 
 	STORAGE_PUBLIC_CLOUD     = "cloud"
 	STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency"
@@ -69,7 +70,7 @@ var (
 	STORAGE_ALL_TYPES     = []string{
 		STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN,
-		STORAGE_NFS,
+		STORAGE_NFS, STORAGE_GPFS,
 	}
 	STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS,
@@ -80,7 +81,15 @@ var (
 		STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA,
 		STORAGE_OPENSTACK_ISCSI, STORAGE_UCLOUD_CLOUD_NORMAL, STORAGE_UCLOUD_CLOUD_SSD,
 		STORAGE_UCLOUD_LOCAL_NORMAL, STORAGE_UCLOUD_LOCAL_SSD, STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK,
+		STORAGE_GPFS,
 	}
 
-	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS}
+	HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL}
+
+	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS}
+
+	SHARED_FILE_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS}
+
+	// 目前来说只支持这些
+	SHARED_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS, STORAGE_RBD}
 )

pkg/apis/compute/vpcs_const.go

@@ -0,0 +1,17 @@
+package compute
+
+const (
+	VPC_STATUS_PENDING       = "pending"
+	VPC_STATUS_AVAILABLE     = "available"
+	VPC_STATUS_FAILED        = "failed"
+	VPC_STATUS_START_DELETE  = "start_delete"
+	VPC_STATUS_DELETING      = "deleting"
+	VPC_STATUS_DELETE_FAILED = "delete_failed"
+	VPC_STATUS_DELETED       = "deleted"
+	VPC_STATUS_UNKNOWN       = "unknown"
+
+	MAX_VPC_PER_REGION = 3
+
+	DEFAULT_VPC_ID = "default"
+	NORMAL_VPC_ID  = "normal" // 没有关联VPC的安全组，统一使用normal
+)

pkg/apis/compute/zones_const.go

@@ -0,0 +1,9 @@
+package compute
+
+const (
+	ZONE_INIT    = "init"
+	ZONE_ENABLE  = "enable"
+	ZONE_DISABLE = "disable"
+	ZONE_SOLDOUT = "soldout"
+	// ZONE_LACK    = "lack"
+)

pkg/apis/image/consts.go

@@ -0,0 +1,26 @@
+package image
+
+type TImageType string
+
+const (
+	// https://docs.openstack.org/glance/pike/user/statuses.html
+	//
+	IMAGE_STATUS_QUEUED     = "queued"
+	IMAGE_STATUS_SAVING     = "saving"
+	IMAGE_STATUS_ACTIVE     = "active"
+	IMAGE_STATUS_CONVERTING = "converting"
+
+	IMAGE_STATUS_DEACTIVATED    = "deactivated"
+	IMAGE_STATUS_KILLED         = "killed"
+	IMAGE_STATUS_DELETED        = "deleted"
+	IMAGE_STATUS_PENDING_DELETE = "pending_delete"
+
+	ImageTypeTemplate = TImageType("image")
+	ImageTypeISO      = TImageType("iso")
+
+	LocalFilePrefix = "file://"
+)
+
+var (
+	ImageDeadStatus = []string{IMAGE_STATUS_DEACTIVATED, IMAGE_STATUS_KILLED, IMAGE_STATUS_DELETED, IMAGE_STATUS_PENDING_DELETE}
+)

pkg/apis/scheduler/api.go

@@ -38,9 +38,6 @@ type ServerConfig struct {
 	Name        string `json:"name"`
 	GuestStatus string `json:"guest_status"`
 
-	// HostId used by migrate
-	HostId string `json:"host_id"`
-
 	// DEPRECATED
 	Metadata       map[string]string `json:"__meta__"`
 	ForGuests      []*ForGuest       `json:"for_guests"`
@@ -56,6 +53,12 @@ type ScheduleInput struct {
 	ScheduleBaseConfig
 
 	ServerConfig
+
+	// HostId used by migrate
+	HostId       string `json:"host_id"`
+	LiveMigrate  bool   `json:"live_migrate"`
+	CpuDesc      string `json:"cpu_desc"`
+	CpuMicrocode string `json:"cpu_microcode"`
 }
 
 func (input ScheduleInput) ToConditionInput() *jsonutils.JSONDict {
@@ -66,19 +69,18 @@ func (input ScheduleInput) ToConditionInput() *jsonutils.JSONDict {
 }
 
 type CandidateDisk struct {
-	Index     int    `json:"index"`
-	StorageId string `json:"storage_id"`
+	Index      int      `json:"index"`
+	StorageIds []string `json:"storage_ids"`
 }
 
 type CandidateResource struct {
-	HostId string           `json:"host_id"`
-	Name   string           `json:"name"`
-	Disks  []*CandidateDisk `json:"disks"`
+	SessionId string           `json:"session_id"`
+	HostId    string           `json:"host_id"`
+	Name      string           `json:"name"`
+	Disks     []*CandidateDisk `json:"disks"`
 
 	// used by backup schedule
 	BackupCandidate *CandidateResource `json:"backup_candidate"`
-	IsMaster        bool               `json:"is_master"`
-	IsSlave         bool               `json:"is_slave"`
 
 	// Error means no candidate found, include reasons
 	Error string `json:"error"`

pkg/appsrv/appsrv.go

@@ -1,9 +1,11 @@
 package appsrv
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"math/rand"
+	"net"
 	"net/http"
 	"os"
 	"strings"
@@ -27,6 +29,7 @@ type Application struct {
 	name              string
 	context           context.Context
 	session           *SWorkerManager
+	readSession       *SWorkerManager
 	systemSession     *SWorkerManager
 	roots             map[string]*RadixNode
 	rootLock          *sync.RWMutex
@@ -60,6 +63,7 @@ func NewApplication(name string, connMax int, db bool) *Application {
 		context:           context.Background(),
 		connMax:           connMax,
 		session:           NewWorkerManager("HttpRequestWorkerManager", connMax, DEFAULT_BACKLOG, db),
+		readSession:       NewWorkerManager("HttpGetRequestWorkerManager", connMax, DEFAULT_BACKLOG, db),
 		systemSession:     NewWorkerManager("InternalHttpRequestWorkerManager", 1, DEFAULT_BACKLOG, false),
 		roots:             make(map[string]*RadixNode),
 		rootLock:          &sync.RWMutex{},
@@ -141,6 +145,13 @@ type loggingResponseWriter struct {
 	status int
 }
 
+func (lrw *loggingResponseWriter) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) {
+	if f, ok := lrw.ResponseWriter.(http.Hijacker); ok {
+		return f.Hijack()
+	}
+	return nil, nil, fmt.Errorf("not a hijacker")
+}
+
 func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	if code < 100 || code >= 600 {
 		log.Errorf("Invalud status code %d, set code to 598", code)
@@ -208,7 +219,7 @@ func (app *Application) handleCORS(w http.ResponseWriter, r *http.Request) bool
 }
 
 func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, rid string) (*SHandlerInfo, *SAppParams) {
-	segs := SplitPath(r.URL.Path)
+	segs := SplitPath(r.URL.EscapedPath())
 	params := make(map[string]string)
 	w.Header().Set("Server", "Yunion AppServer/Go/2018.4")
 	w.Header().Set("X-Frame-Options", "SAMEORIGIN")
@@ -220,7 +231,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 		if ok {
 			fw := newResponseWriterChannel(w)
 			worker := make(chan *SWorker)
-			to := hand.processTimeout
+			to := hand.FetchProcessTimeout(r)
 			if to == 0 {
 				to = app.processTimeout
 			}
@@ -228,7 +239,11 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 			defer cancel()
 			session := hand.workerMan
 			if session == nil {
-				session = app.session
+				if r.Method == "GET" || r.Method == "HEAD" {
+					session = app.readSession
+				} else {
+					session = app.session
+				}
 			}
 			appParams := hand.GetAppParams(params, segs)
 			appParams.Request = r

pkg/appsrv/handlerinfo.go

@@ -27,6 +27,14 @@ type SHandlerInfo struct {
 	skipLog        bool
 }
 
+func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration {
+	if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
+		return time.Hour * 2
+	} else {
+		return this.processTimeout
+	}
+}
+
 func (this *SHandlerInfo) GetName(params map[string]string) string {
 	if len(this.name) > 0 {
 		return this.name

pkg/appsrv/response.go

@@ -1,8 +1,10 @@
 package appsrv
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -66,6 +68,18 @@ func (w *responseWriterChannel) Flush() {
 	}
 }
 
+// Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter
+// and a Hijacker.
+func (w *responseWriterChannel) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) {
+	if w.isClosed {
+		return nil, nil, fmt.Errorf("response stream has been closed")
+	}
+	if f, ok := w.backend.(http.Hijacker); ok {
+		return f.Hijack()
+	}
+	return nil, nil, fmt.Errorf("not a hijacker")
+}
+
 func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWorker) interface{} {
 	var err error
 	var worker *SWorker

pkg/appsrv/workers.go

@@ -208,7 +208,7 @@ func (wm *SWorkerManager) scheduleWithLock() {
 			log.Debugf("no enough worker, add new worker %s", worker)
 		}
 		go worker.run()
-	} else {
+	} else if wm.queue.Size() > 10 {
 		log.Warningf("[%s] BUSY activeWork %d detachedWork %d max %d queue: %d", wm, wm.ActiveWorkerCount(), wm.DetachedWorkerCount(), wm.workerCount, wm.queue.Size())
 	}
 }

pkg/baremetal/manager.go

@@ -677,15 +677,24 @@ func (b *SBaremetalInstance) GetNotifyUrl() string {
 }
 
 func (b *SBaremetalInstance) GetTFTPResponse() string {
-	return fmt.Sprintf(
-		`default start
+	resp := `default start
 serial 1 115200
+
 label start
     menu label ^Start
     menu default
-    kernel kernel
-    append initrd=initramfs token=%s url=%s`,
-		auth.GetTokenString(), b.GetNotifyUrl())
+`
+
+	if b.NeedPXEBoot() {
+		resp += "    kernel kernel\n"
+		resp += fmt.Sprintf("    append initrd=initramfs token=%s url=%s",
+			auth.GetTokenString(), b.GetNotifyUrl())
+	} else {
+		resp += "    COM32 chain.c32\n"
+		resp += "    APPEND hd0 0\n"
+	}
+	log.Infof("%s", resp)
+	return resp
 }
 
 func (b *SBaremetalInstance) GetTaskQueue() *tasks.TaskQueue {
@@ -920,6 +929,7 @@ func (b *SBaremetalInstance) DoPXEBoot() error {
 	return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId())
 }
 
+/*
 func (b *SBaremetalInstance) DoDiskBoot() error {
 	log.Infof("Do DISK Boot ........., wait")
 	b.ClearSSHConfig()
@@ -929,6 +939,7 @@ func (b *SBaremetalInstance) DoDiskBoot() error {
 	}
 	return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId())
 }
+*/
 
 func (b *SBaremetalInstance) GetPowerStatus() (string, error) {
 	ipmiCli := b.GetIPMITool()
@@ -1005,7 +1016,7 @@ func (b *SBaremetalInstance) StartBaremetalResetBMCTask(userCred mcclient.TokenC
 }
 
 func (b *SBaremetalInstance) DelayedServerReset(_ jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	err := b.DoDiskBoot()
+	err := b.DoPXEBoot()
 	return nil, err
 }
 

pkg/baremetal/options/options.go

@@ -18,7 +18,8 @@ type BaremetalOptions struct {
 	DhcpRenewalTime        int    `default:"67108864" help:"DHCP renewal time in seconds"` // 0x4000000
 	EnableGeneralGuestDhcp bool   `default:"false" help:"Enable DHCP service for general guest, e.g. those on VMware ESXi or Xen"`
 	ForceDhcpProbeIpmi     bool   `default:"false" help:"Force DHCP probe IPMI interface network connection"`
-	TftpMaxTimeoutRetries  int    `default:"20" help:"Maximal tftp timeout retries, default is 20"`
+	TftpBlockSizeInBytes   int    `default:"1024" help:"tftp block size, default is 1024"`
+	TftpMaxTimeoutRetries  int    `default:"50" help:"Maximal tftp timeout retries, default is 50"`
 	LengthyWorkerCount     int    `default:"8" help:"Parallel worker count for lengthy tasks"`
 	ShortWorkerCount       int    `default:"8" help:"Parallel worker count for short-lived tasks"`
 

pkg/baremetal/pxe/dhcp.go

@@ -153,12 +153,14 @@ func (req *dhcpRequest) fetchConfig(session *mcclient.ClientSession) (*dhcp.Resp
 		if err != nil {
 			return nil, err
 		}
-		if req.baremetalInstance.NeedPXEBoot() {
-			return req.baremetalInstance.GetPXEDHCPConfig(req.ClientArch)
-		}
+		// always response PXE request
+		// let bootloader decide boot local or remote
+		// if req.baremetalInstance.NeedPXEBoot() {
+		return req.baremetalInstance.GetPXEDHCPConfig(req.ClientArch)
+		// }
 		// ignore
-		log.Warningf("No need to pxeboot, ignore the request ...(mac:%s guid:%s)", req.ClientMac, req.ClientGuid)
-		return nil, nil
+		// log.Warningf("No need to pxeboot, ignore the request ...(mac:%s guid:%s)", req.ClientMac, req.ClientGuid)
+		// return nil, nil
 	} else {
 		// handle normal DHCP request
 		bmInstance := req.baremetalManager.GetBaremetalByMac(req.ClientMac)
@@ -221,12 +223,11 @@ func (req *dhcpRequest) findNetworkConf(session *mcclient.ClientSession, filterU
 
 func (req *dhcpRequest) findBaremetalsOfAnyMac(session *mcclient.ClientSession, isBaremetal bool) (*modules.ListResult, error) {
 	params := jsonutils.NewDict()
-	params.Add(jsonutils.NewString(api.HOST_TYPE_BAREMETAL), "host_type")
 	params.Add(jsonutils.NewString(req.ClientMac.String()), "any_mac")
 	if isBaremetal {
 		params.Add(jsonutils.JSONTrue, "is_baremetal")
 	} else {
-		params.Add(jsonutils.NewString("baremetal"), "host_type")
+		params.Add(jsonutils.NewString(api.HOST_TYPE_BAREMETAL), "host_type")
 	}
 	return modules.Hosts.List(session, params)
 }

pkg/baremetal/pxe/tftp.go

@@ -12,6 +12,7 @@ import (
 
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/baremetal/options"
 	"yunion.io/x/onecloud/pkg/util/tftp"
 )
 
@@ -124,6 +125,9 @@ func (s *Server) serveTFTP(l net.PacketConn, handler *TFTPHandler) error {
 		InfoLog:     func(msg string) { log.Debugf("TFTP msg: %s", msg) },
 		TransferLog: handler.transferLog,
 		Dial:        bindDial,
+
+		MaxBlockSize:  int64(options.Options.TftpBlockSizeInBytes),
+		WriteAttempts: options.Options.TftpMaxTimeoutRetries,
 	}
 	err := ts.Serve(l)
 	if err != nil {

pkg/baremetal/tasks/base.go

@@ -269,25 +269,15 @@ func (self *SBaremetalTaskBase) EnsurePowerShutdown(soft bool) error {
 	return nil
 }
 
-func (self *SBaremetalTaskBase) EnsurePowerUp(bootdev string) error {
-	log.Infof("EnsurePowerUp: bootdev=%s", bootdev)
-	var bootFunc func() error = nil
-	switch bootdev {
-	case "pxe":
-		bootFunc = self.Baremetal.DoPXEBoot
-	case "disk":
-		bootFunc = self.Baremetal.DoDiskBoot
-	}
-	if bootFunc == nil {
-		return fmt.Errorf("No boot func %s found", bootdev)
-	}
+func (self *SBaremetalTaskBase) EnsurePowerUp() error {
+	log.Infof("EnsurePowerUp: bootdev=pxe")
 	status, err := self.Baremetal.GetPowerStatus()
 	if err != nil {
 		return err
 	}
 	for status == "" || status == types.POWER_STATUS_OFF {
 		if status == types.POWER_STATUS_OFF {
-			err = bootFunc()
+			err = self.Baremetal.DoPXEBoot()
 			if err != nil {
 				return err
 			}
@@ -362,7 +352,7 @@ func (self *SBaremetalPXEBootTaskBase) InitPXEBootTask(pxeBootTask IPXEBootTask,
 	if err := self.EnsurePowerShutdown(false); err != nil {
 		return self, fmt.Errorf("EnsurePowerShutdown: %v", err)
 	}
-	if err := self.EnsurePowerUp("pxe"); err != nil {
+	if err := self.EnsurePowerUp(); err != nil {
 		return self, fmt.Errorf("EnsurePowerUp to pxe: %v", err)
 	}
 	// this stage will be called by baremetalInstance when pxe start notify
@@ -392,7 +382,7 @@ func (self *SBaremetalPXEBootTaskBase) WaitForShutdown(ctx context.Context, args
 }
 
 func (self *SBaremetalPXEBootTaskBase) OnStopComplete(ctx context.Context, args interface{}) error {
-	err := self.EnsurePowerUp("pxe")
+	err := self.EnsurePowerUp()
 	if err != nil {
 		return err
 	}

pkg/baremetal/tasks/basedeploy.go

@@ -66,7 +66,7 @@ func (self *SBaremetalServerBaseDeployTask) OnPXEBoot(ctx context.Context, term
 			return err
 		}
 		if onFinishAction == "restart" {
-			err = self.EnsurePowerUp("disk")
+			err = self.EnsurePowerUp()
 			if err != nil {
 				return err
 			}

pkg/baremetal/tasks/baseprepare.go

@@ -108,6 +108,7 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 				log.Errorf("Lan channel %d MAC address is empty", lanChannel)
 				continue
 			}
+
 			ipmiNic := &types.SNicDevInfo{
 				Mac:   conf.Mac,
 				Up:    false,
@@ -118,10 +119,12 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 			rootId := ipmitool.GetRootId(ipmiSysInfo)
 			err = ipmitool.SetLanUserPasswd(sshIPMI, lanChannel, rootId, ipmiUser, ipmiPasswd)
 			if err != nil {
+				// ignore the error
 				log.Errorf("Lan channel %d set user password error: %v", lanChannel, err)
 			}
 			err = ipmitool.EnableLanAccess(sshIPMI, lanChannel)
 			if err != nil {
+				// ignore the error
 				log.Errorf("Lan channel %d enable lan access error: %v", lanChannel, err)
 			}
 
@@ -134,55 +137,20 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 			}
 			if len(tryAddrs) > 0 && !o.Options.ForceDhcpProbeIpmi {
 				for _, tryAddr := range tryAddrs {
-					log.Infof("IP addr found in IPMI config, try use %s as IPMI address", tryAddr)
-					ipConf, err := task.getIPMIIPConfig(tryAddr)
-					if err != nil {
-						log.Errorf("Failed to get IPMI ipconfig for %s", tryAddr)
-						continue
-					}
-					err = ipmitool.SetLanStatic(sshIPMI, lanChannel, ipConf.IPAddr, ipConf.Netmask, ipConf.Gateway)
-					if err != nil {
-						log.Errorf("Failed to set IPMI static net config %#v for %s", *ipConf, tryAddr)
-						continue
-					}
-					time.Sleep(1 * time.Second)
-					conf, err = ipmitool.GetLanConfig(sshIPMI, lanChannel)
-					if err != nil {
-						log.Errorf("Failed to get lan config after set static network: %v", err)
-						continue
-					}
-					log.Infof("Get lan config %#v", *conf)
-					if conf.IPAddr == "" || conf.IPAddr != tryAddr {
-						log.Errorf("Failed to set ipmi lan channel %d static ipaddr", lanChannel)
-						continue
-					}
-					rmcpIPMI := ipmitool.NewLanPlusIPMI(tryAddr, ipmiUser, ipmiPasswd)
-					conf2, err := ipmitool.GetLanConfig(rmcpIPMI, lanChannel)
-					if err != nil {
-						log.Errorf("Failed to get lan channel %d config use RMCP mode: %v", lanChannel, err)
-						continue
-					}
-					if len(conf2.Mac) != 0 &&
-						conf2.Mac.String() == conf.Mac.String() &&
-						conf2.IPAddr != "" && conf2.IPAddr == tryAddr {
-						// make sure the ipaddr is a IPMI address
-						// enable the netif
-						if err := task.sendNicInfo(ipmiNic, -1, types.NIC_TYPE_IPMI, false, tryAddr); err != nil {
-							log.Errorf("Fail to set existing BMC IP address to %s", tryAddr)
-						} else {
-							ipmiInfo.IpAddr = tryAddr
-							ipmiLanChannel = lanChannel
-							break
-						}
-					} else {
-						log.Errorf("Use RMCP mode get invalid lan config: %#v", conf2)
-					}
-					if ipmiLanChannel >= 0 {
-						// found and set config on lanChannel
+					tryResult := task.tryLocalIpmiAddr(sshIPMI, ipmiNic, lanChannel,
+						ipmiUser, ipmiPasswd, tryAddr)
+					if tryResult {
+						ipmiInfo.IpAddr = tryAddr
+						ipmiLanChannel = lanChannel
 						break
 					}
 				}
+				if ipmiLanChannel >= 0 {
+					// found and set config on lanChannel
+					break
+				}
 			}
+
 			if len(tryAddrs) > 0 {
 				task.baremetal.SetExistingIPMIIPAddr(tryAddrs[0])
 			}
@@ -280,14 +248,15 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 	if err := task.sendStorageInfo(size); err != nil {
 		log.Errorf("sendStorageInfo error: %v", err)
 	}
-	for i := range nicsInfo {
-		if nicsInfo[i].Mac.String() == adminNic.GetMac().String() {
-			if i != 0 {
-				nicsInfo = append(nicsInfo[i:], nicsInfo[0:i]...)
-			}
-			break
-		}
-	}
+	// XXX do not change nic order anymore
+	// for i := range nicsInfo {
+	// 	if nicsInfo[i].Mac.String() == adminNic.GetMac().String() {
+	// 		if i != 0 {
+	// 			nicsInfo = append(nicsInfo[i:], nicsInfo[0:i]...)
+	// 		}
+	// 		break
+	// 	}
+	// }
 	err = task.removeAllNics()
 	if err != nil {
 		return err
@@ -311,6 +280,70 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error {
 	return nil
 }
 
+func (task *sBaremetalPrepareTask) tryLocalIpmiAddr(sshIPMI *ipmitool.SSHIPMI, ipmiNic *types.SNicDevInfo, lanChannel int, ipmiUser, ipmiPasswd, tryAddr string) bool {
+	log.Infof("IP addr found in IPMI config, try use %s as IPMI address", tryAddr)
+	ipConf, err := task.getIPMIIPConfig(tryAddr)
+	if err != nil {
+		log.Errorf("Failed to get IPMI ipconfig for %s", tryAddr)
+		return false
+	}
+	err = ipmitool.SetLanStatic(sshIPMI, lanChannel, ipConf.IPAddr, ipConf.Netmask, ipConf.Gateway)
+	if err != nil {
+		log.Errorf("Failed to set IPMI static net config %#v for %s", *ipConf, tryAddr)
+		return false
+	}
+
+	var conf *types.SIPMILanConfig
+
+	tried := 0
+	maxTries := 5
+	time.Sleep(2 * time.Second)
+	for tried = 0; tried < maxTries; tried += 1 {
+		conf, err = ipmitool.GetLanConfig(sshIPMI, lanChannel)
+		if err != nil {
+			log.Errorf("Failed to get lan config after set static network: %v", err)
+			continue
+		}
+		log.Infof("Get lan config %#v", *conf)
+		if conf.IPAddr == "" || conf.IPAddr != tryAddr {
+			log.Errorf("Failed to set ipmi lan channel %d static ipaddr", lanChannel)
+			continue
+		}
+		break
+	}
+	if tried >= maxTries {
+		log.Errorf("Failed to get lan config after %s tries", tried)
+		return false
+	}
+	rmcpIPMI := ipmitool.NewLanPlusIPMI(tryAddr, ipmiUser, ipmiPasswd)
+	for tried = 0; tried < maxTries; tried += 1 {
+		conf2, err := ipmitool.GetLanConfig(rmcpIPMI, lanChannel)
+		if err != nil {
+			log.Errorf("Failed to get lan channel %d config use RMCP mode: %v", lanChannel, err)
+			continue
+		}
+		if len(conf2.Mac) != 0 &&
+			conf2.Mac.String() == conf.Mac.String() &&
+			conf2.IPAddr != "" && conf2.IPAddr == tryAddr {
+			break
+		} else {
+			log.Errorf("fail to rmcp get IPMI ip config %v", conf2)
+			time.Sleep(5 * time.Second)
+		}
+	}
+	if tried < maxTries {
+		// make sure the ipaddr is a IPMI address
+		// enable the netif
+		err := task.sendNicInfo(ipmiNic, -1, types.NIC_TYPE_IPMI, false, tryAddr)
+		if err != nil {
+			log.Errorf("Fail to set existing BMC IP address to %s", tryAddr)
+		} else {
+			return true
+		}
+	}
+	return false
+}
+
 func (task *sBaremetalPrepareTask) getIPMIUserPasswd(sysInfo *types.SIPMISystemInfo) (string, string, string) {
 	var (
 		ipmiUser   string

pkg/baremetal/tasks/interface.go

@@ -34,7 +34,7 @@ type IBaremetal interface {
 	GetPowerStatus() (string, error)
 	DoPowerShutdown(soft bool) error
 	DoPXEBoot() error
-	DoDiskBoot() error
+	// DoDiskBoot() error
 
 	RemoveServer()
 }

pkg/baremetal/tasks/start.go

@@ -23,8 +23,8 @@ func NewBaremetalServerStartTask(
 	self := &SBaremetalServerStartTask{
 		SBaremetalTaskBase: baseTask,
 	}
-	if err := self.Baremetal.DoDiskBoot(); err != nil {
-		return nil, fmt.Errorf("DoDiskBoot: %v", err)
+	if err := self.Baremetal.DoPXEBoot(); err != nil {
+		return nil, fmt.Errorf("DoPXEBoot: %v", err)
 	}
 	self.SetStage(self.WaitForStart)
 	ExecuteTask(self, nil)

pkg/baremetal/tasks/unmaintenance.go

@@ -28,7 +28,7 @@ func NewBaremetalUnmaintenanceTask(
 		if err != nil {
 			return task, fmt.Errorf("EnsurePowerShutdown hard: %v", err)
 		}
-		err = task.EnsurePowerUp("disk")
+		err = task.EnsurePowerUp()
 		if err != nil {
 			return task, fmt.Errorf("EnsurePowerUp disk: %v", err)
 		}

pkg/baremetal/utils/disktool/disktool.go

@@ -90,7 +90,8 @@ func (p *Partition) Format(fs string, uuid string) error {
 		cmd = []string{"/usr/sbin/mkfs.ext3"}
 		cmdUUID = []string{"/usr/sbin/tune2fs", "-U", uuid}
 	case "ext4":
-		cmd = []string{"/usr/sbin/mkfs.ext4", "-O", "^64bit", "-E", "lazy_itable_init=1"}
+		// for baremetal, force 64bit support large disks
+		cmd = []string{"/usr/sbin/mkfs.ext4", "-O", "64bit", "-E", "lazy_itable_init=1"}
 		cmdUUID = []string{"/usr/sbin/tune2fs", "-U", uuid}
 	case "ext4dev":
 		cmd = []string{"/usr/sbin/mkfs.ext4dev", "-E", "lazy_itable_init=1"}

pkg/cloudcommon/agent/doc.go

@@ -0,0 +1 @@
+package agent // import "yunion.io/x/onecloud/pkg/cloudcommon/agent"

pkg/cloudcommon/cmdline/helper.go

@@ -310,10 +310,11 @@ func FetchScheduleInputByJSON(obj jsonutils.JSONObject) (*scheduler.ScheduleInpu
 	if err != nil {
 		return nil, err
 	}
+	conf := &input.ServerConfig
 	if obj.Contains("scheduler") {
 		obj, _ = obj.Get("scheduler")
+		obj.Unmarshal(conf)
 	}
-	conf := &input.ServerConfig
 	conf.ServerConfigs, err = FetchServerConfigsByJSON(obj)
 	if err != nil {
 		return nil, err

pkg/cloudcommon/cmdline/parser.go

@@ -95,6 +95,8 @@ func ParseDiskConfig(diskStr string, idx int) (*compute.DiskConfig, error) {
 			diskConfig.Mountpoint = p
 		} else if p == "autoextend" {
 			diskConfig.SizeMb = -1
+		} else if utils.IsInStringArray(p, compute.STORAGE_ALL_TYPES) {
+			diskConfig.Backend = p
 		} else if strings.HasPrefix(p, "snapshot-") {
 			// HACK: use snapshot creat disk format snapshot-id
 			// example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b

pkg/cloudcommon/db/adminsharablevirtual.go

@@ -9,7 +9,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SAdminSharableVirtualResourceBase struct {
@@ -138,6 +137,5 @@ func (model *SAdminSharableVirtualResourceBase) setInfo(ctx context.Context,
 		return err
 	}
 	OpsLog.LogEvent(model, ACT_UPDATE, diff, userCred)
-	logclient.AddActionLogWithContext(ctx, model, logclient.ACT_UPDATE, diff, userCred, true)
 	return err
 }

pkg/cloudcommon/db/db_dispatcher.go

@@ -24,8 +24,8 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/httputils"
-	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 type DBModelDispatcher struct {
@@ -310,13 +310,28 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem
 	return q, nil
 }
 
-func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery, query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) {
-	listF := listFields(manager, userCred)
-	fieldFilter := jsonutils.GetQueryStringArray(query, "field")
-	if len(fieldFilter) > 0 && IsAdminAllowList(userCred, manager) {
-		// only sysadmin can specify list Fields
-		listF = fieldFilter
+func mergeFields(metaFields, queryFields []string, isAdmin bool) stringutils2.SSortedStrings {
+	meta := stringutils2.NewSortedStrings(metaFields)
+	if len(queryFields) == 0 {
+		return meta
 	}
+
+	query := stringutils2.NewSortedStrings(queryFields)
+	_, mAndQ, qNoM := stringutils2.Split(meta, query)
+
+	if !isAdmin {
+		return mAndQ
+	}
+
+	// only sysadmin can specify list Fields
+	return stringutils2.Merge(mAndQ, qNoM)
+}
+
+func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery, query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) {
+	metaFields := listFields(manager, userCred)
+	fieldFilter := jsonutils.GetQueryStringArray(query, "field")
+	listF := mergeFields(metaFields, fieldFilter, IsAdminAllowList(userCred, manager))
+
 	showDetails := false
 	showDetailsJson, _ := query.Get("details")
 	if showDetailsJson != nil {
@@ -324,15 +339,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 	} else {
 		showDetails = true
 	}
-	item, err := NewModelObject(manager)
-	if err != nil {
-		return nil, err
-	}
-	itemInitValue := reflect.Indirect(reflect.ValueOf(item))
-	item, err = NewModelObject(manager)
-	if err != nil {
-		return nil, err
-	}
+	items := make([]IModel, 0)
 	results := make([]jsonutils.JSONObject, 0)
 	rows, err := q.Rows()
 	if err != nil {
@@ -340,8 +347,10 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 	}
 	defer rows.Close()
 	for rows.Next() {
-		itemValue := reflect.Indirect(reflect.ValueOf(item))
-		itemValue.Set(itemInitValue)
+		item, err := NewModelObject(manager)
+		if err != nil {
+			return nil, err
+		}
 		extraData := jsonutils.NewDict()
 		if query.Contains("export_keys") {
 			RowMap, err := q.Row2Map(rows)
@@ -363,12 +372,8 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 			}
 		}
 
-		jsonData := jsonutils.Marshal(item)
-		jsonDict, ok := jsonData.(*jsonutils.JSONDict)
-		if !ok {
-			return nil, fmt.Errorf("invalid model data structure, not a dict")
-		}
-		jsonDict = jsonDict.CopyIncludes(listF...)
+		jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict)
+		jsonDict = jsonDict.CopyIncludes([]string(listF)...)
 		jsonDict.Update(extraData)
 		if showDetails && !query.Contains("export_keys") {
 			extraDict := item.GetCustomizeColumns(ctx, userCred, query)
@@ -378,6 +383,16 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 			jsonDict = getModelExtraDetails(item, ctx, jsonDict)
 		}
 		results = append(results, jsonDict)
+		items = append(items, item)
+	}
+	if showDetails && !query.Contains("export_keys") {
+		extraRows := manager.FetchCustomizeColumns(ctx, userCred, query, items, stringutils2.NewSortedStrings(fieldFilter))
+		// log.Debugf("manager.FetchCustomizeColumns: %s %s", extraRows, listF)
+		if len(extraRows) == len(results) {
+			for i := range results {
+				results[i].(*jsonutils.JSONDict).Update(extraRows[i])
+			}
+		}
 	}
 	return results, nil
 }
@@ -446,6 +461,13 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if totalCnt > maxLimit && (limit <= 0 || limit > maxLimit) {
 		limit = maxLimit
 	}
+
+	// export data only
+	exportLimit, err := query.Int("export_limit")
+	if query.Contains("export_keys") && err == nil {
+		limit = exportLimit
+	}
+
 	orderBy := jsonutils.GetQueryStringArray(queryDict, "order_by")
 	if len(orderBy) == 0 {
 		colSpec := manager.TableSpec().ColumnSpec("id")
@@ -551,7 +573,7 @@ func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.J
 		log.Errorf("Fail to list items: %s", err)
 		return nil, httperrors.NewGeneralError(err)
 	}
-	if userCred.HasSystemAdminPrivilege() && dispatcher.modelManager.ListSkipLog(ctx, userCred, query) {
+	if userCred != nil && userCred.HasSystemAdminPrivilege() && dispatcher.modelManager.ListSkipLog(ctx, userCred, query) {
 		appParams := appsrv.AppContextGetParams(ctx)
 		if appParams != nil {
 			appParams.SkipLog = true
@@ -604,20 +626,27 @@ func getItemDetails(manager IModelManager, item IModel, ctx context.Context, use
 	extraDict, err := item.GetExtraDetails(ctx, userCred, query)
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
-	} else if extraDict != nil {
-		jsonData := jsonutils.Marshal(item)
-		jsonDict, ok := jsonData.(*jsonutils.JSONDict)
-		if !ok {
-			return nil, fmt.Errorf("fail to convert model to json")
-		}
-		jsonDict = jsonDict.CopyIncludes(GetDetailFields(manager, userCred)...)
-		jsonDict.Update(extraDict)
-		jsonDict = getModelExtraDetails(item, ctx, jsonDict)
-		return jsonDict, nil
-	} else {
+	}
+	if extraDict == nil {
 		// override GetExtraDetails
 		return nil, nil
 	}
+
+	metaFields := GetDetailFields(manager, userCred)
+	fieldFilter := jsonutils.GetQueryStringArray(query, "field")
+	getFields := mergeFields(metaFields, fieldFilter, IsAdminAllowGet(userCred, item))
+
+	jsonDict := jsonutils.Marshal(item).(*jsonutils.JSONDict)
+	jsonDict = jsonDict.CopyIncludes(getFields...)
+	jsonDict.Update(extraDict)
+	jsonDict = getModelExtraDetails(item, ctx, jsonDict)
+
+	extraRows := manager.FetchCustomizeColumns(ctx, userCred, query, []IModel{item}, stringutils2.NewSortedStrings(fieldFilter))
+	if len(extraRows) == 1 {
+		jsonDict.Update(extraRows[0])
+	}
+
+	return jsonDict, nil
 }
 
 func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, property string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
@@ -937,7 +966,6 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 	}()
 
 	OpsLog.LogEvent(model, ACT_CREATE, model.GetShortDesc(ctx), userCred)
-	logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, "", userCred, true)
 	dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, query, data)
 	return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query)
 }
@@ -1218,13 +1246,11 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 
 	if err != nil {
 		log.Errorf("validate update condition error: %s", err)
-		logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
 	dataDict, ok := data.(*jsonutils.JSONDict)
 	if !ok {
-		logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, "Invalid data JSONObject", userCred, false)
 		return nil, httperrors.NewInternalServerError("Invalid data JSONObject")
 	}
 
@@ -1232,7 +1258,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 	if len(name) > 0 {
 		err = alterNameValidator(item, name)
 		if err != nil {
-			logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 			return nil, err
 		}
 	}
@@ -1241,7 +1266,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 	if err != nil {
 		errMsg := fmt.Sprintf("validate update data error: %s", err)
 		log.Errorf(errMsg)
-		logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, errMsg, userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
@@ -1252,7 +1276,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		err = filterData.Unmarshal(item)
 		if err != nil {
 			errMsg := fmt.Sprintf("unmarshal fail: %s", err)
-			logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, errMsg, userCred, false)
 			log.Errorf(errMsg)
 			return httperrors.NewGeneralError(err)
 		}
@@ -1264,7 +1287,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		return nil, httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(item, ACT_UPDATE, diff, userCred)
-	logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, diff, userCred, true)
 
 	item.PostUpdate(ctx, userCred, query, data)
 
@@ -1308,11 +1330,9 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 	if err != nil {
 		msg := fmt.Sprintf("save update error %s", err)
 		log.Errorf(msg)
-		// logclient.AddActionLogWithContext(ctx, item, logclient.ACT_DELETE, msg, userCred, false)
 		return httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(ctx), userCred)
-	logclient.AddSimpleActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(ctx), userCred, true)
 	return nil
 }
 

pkg/cloudcommon/db/enabledstatusstandalone.go

@@ -39,7 +39,7 @@ func (self *SEnabledStatusStandaloneResourceBase) PerformEnable(ctx context.Cont
 			return nil, err
 		}
 		OpsLog.LogEvent(self, ACT_ENABLE, "", userCred)
-		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_ENABLE, nil, userCred, true)
+		logclient.AddSimpleActionLog(self, logclient.ACT_ENABLE, nil, userCred, true)
 	}
 	return nil, nil
 }
@@ -59,7 +59,7 @@ func (self *SEnabledStatusStandaloneResourceBase) PerformDisable(ctx context.Con
 			return nil, err
 		}
 		OpsLog.LogEvent(self, ACT_DISABLE, "", userCred)
-		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_DISABLE, nil, userCred, true)
+		logclient.AddSimpleActionLog(self, logclient.ACT_DISABLE, nil, userCred, true)
 	}
 	return nil, nil
 }