Merge pull request #14970 from swordqiu/automated-cherry-pick-of-#14967-upstream-release-3.7

Automated cherry pick of #14967: fix: set vfs_cache_pressure to 350

Makefile

@@ -268,7 +268,8 @@ dep:
 	@$(MAKE) mod
 
 mod:
-	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
+	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v sqlchemy))
+	go get -d yunion.io/x/sqlchemy@v1.0.0
 	go mod tidy
 	go mod vendor -v
 
@@ -330,7 +331,7 @@ image:
 .PHONY: image
 
 image-telegraf-raid-plugin:
-	VERSION=release-1.6.1 ARCH=all make image telegraf-raid-plugin
+	VERSION=release-1.6.3 ARCH=all make image telegraf-raid-plugin
 
 %:
 	@:

Makefile.common.mk

@@ -7,8 +7,8 @@ endif
 ModBaseName:=$(notdir $(ModName))
 
 DockerImageRegistry?=registry.cn-beijing.aliyuncs.com
-DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.0-5
-DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-4
+DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.1-5
+DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-5
 
 
 EnvIf=$(if $($(1)),$(1)=$($(1)))

build/docker/Dockerfile.ansibleserver

@@ -1,7 +1,7 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.3
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.5
 
 # install playbook and telegraf install pkg
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.2.1 /opt/yunion/playbook /opt/yunion/playbook
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.2.1 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/playbook /opt/yunion/playbook
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
 
 ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/apigateway /opt/yunion/bin/apigateway
+ADD ./_output/alpine-build/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.baremetal-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.1
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.3
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.baremetal-base

@@ -1,7 +1,8 @@
 FROM --platform=linux/amd64 registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
-RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21060506.x86_64.rpm
+RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
+#RUN yum install -y http://192.168.23.50:8083/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
 

build/docker/Dockerfile.climc

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:20210701
+FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:20210901
 
 ADD ./build/climc/root/opt /opt
 

build/docker/Dockerfile.climc-base

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
@@ -8,7 +8,7 @@ RUN mkdir -p /opt/yunion/bin
 
 RUN echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >>/etc/apk/repositories
 
-RUN apk add --no-cache bash bash-completion tzdata ca-certificates vim ipmitool kubectl && \
+RUN apk add --no-cache bash bash-completion tzdata ca-certificates vim ipmitool kubectl ceph-common && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/cloudevent /opt/yunion/bin/cloudevent
+ADD ./_output/alpine-build/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudid

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/cloudid/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/cloudid /opt/yunion/bin/cloudid

build/docker/Dockerfile.cloudmon

@@ -0,0 +1,8 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+RUN mkdir -p /etc/yunion/data/
+
+ADD   ./_output/alpine-build/bin/cloudmon \
+      /opt/yunion/bin/

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/cloudnet /opt/yunion/bin/cloudnet
+ADD ./_output/alpine-build/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.cloudproxy

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/cloudproxy /opt/yunion/bin/cloudproxy

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/devtool /opt/yunion/bin/devtool
+ADD ./_output/alpine-build/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 

build/docker/Dockerfile.file-repo

@@ -11,10 +11,10 @@ RUN set -x \
 
 # install default playbook and install pkg
 Run mkdir -p /opt/yunion/ansible-install-pkg
-Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.5.0~yn-1.aarch64.rpm -P /opt/yunion/ansible-install-pkg
-Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.5.0~yn-1.x86_64.rpm -P /opt/yunion/ansible-install-pkg
-Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.5.0~yn-1_amd64.deb -P /opt/yunion/ansible-install-pkg
-Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.5.0~yn-1_arm64.deb -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.19.2-yn~fe11a96b-0.aarch64.rpm -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.19.2-yn~fe11a96b-0.x86_64.rpm -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.19.2-yn~fe11a96b-0_amd64.deb -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.19.2-yn~fe11a96b-0_arm64.deb -P /opt/yunion/ansible-install-pkg
 
 Run mkdir -p /opt/yunion/playbook
 Run mkdir /opt/yunion/playbook/monitor-agent

build/docker/Dockerfile.glance

@@ -1,7 +1,7 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/torrent:20200820
+FROM registry.cn-beijing.aliyuncs.com/yunionio/torrent:20210815.0
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
 RUN apk add --no-cache s3fs-fuse --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
 RUN mkdir -p /opt/yunion/bin
-ADD ./_output/alpine-build/bin/glance /opt/yunion/bin/glance
+ADD ./_output/alpine-build/bin/glance /opt/yunion/bin/glance

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/keystone /opt/yunion/bin/keystone
+ADD ./_output/alpine-build/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/logger /opt/yunion/bin/logger

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/monitor/root/opt/ /opt/
-ADD ./_output/alpine-build/bin/monitor /opt/yunion/bin/monitor
+ADD ./_output/alpine-build/bin/monitor /opt/yunion/bin/monitor

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -1,4 +1,5 @@
-FROM alpine:3.11
+# TODO: use 3.14 will lead 'apk add qemu-img fail on arm64'
+FROM alpine:3.13.5
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.region-dns

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/region-dns /opt/yunion/bin/region-dns

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/s3gateway /opt/yunion/bin/s3gateway
+ADD ./_output/alpine-build/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/scheduler /opt/yunion/bin/scheduler
+ADD ./_output/alpine-build/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.telegraf-raid-plugin

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.1
 
 ADD ./_output/centos-build/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/Dockerfile.torrent

@@ -1,15 +1,15 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
+MAINTAINER "Zexi Li <zexi.li@icloud.com>"
+
+# HACK: use v3.14 alpine mirror to upgrade qemu-img
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/main > /etc/apk/repositories
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/community >> /etc/apk/repositories
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates qemu-img && \
+    apk add --no-cache qemu-img && \
     rm -rf /var/cache/apk/*
 
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV TZ Asia/Shanghai
-
 # add executable file torrent
 # make cmd/torrent
 RUN mkdir -p /opt/yunion/bin

build/docker/Dockerfile.yunionconf

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/yunionconf /opt/yunion/bin/yunionconf

build/docker/Makefile

@@ -7,14 +7,29 @@ debian10-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/debian10-base:1.0 -f ./Dockerfile.debian-base .
 
-CLIMC_BASE_VERSION = 20210701
+ONECLOUD_BASE_VERSION = v0.3.5
+
+onecloud-base:
+	$(DOCKER_BUILDX)/onecloud-base:$(ONECLOUD_BASE_VERSION) -f ./Dockerfile.onecloud-base .
+
+ANSIBLESERVER_BASE = v1.0.5
+
+ansibleserver-base:
+	$(DOCKER_BUILDX)/ansibleserver-base:$(ANSIBLESERVER_BASE) -f ./Dockerfile.ansibleserver-base .
+
+CLIMC_BASE_VERSION = 20210901
 
 climc-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/climc-base:$(CLIMC_BASE_VERSION) -f ./Dockerfile.climc-base .
 
-BAREMETAL_BASE_VERSION = v0.3.1
+BAREMETAL_BASE_VERSION = v0.3.3
 
 baremetal-base:
 	$(DOCKER_BUILDX)/baremetal-base:$(BAREMETAL_BASE_VERSION) -f ./Dockerfile.baremetal-base .
 	#docker push $(REGISTRY)/baremetal-base:$(BAREMETAL_BASE_VERSION)
+
+FILE_REPO_VERSION = v0.3.3
+
+file-repo:
+	$(DOCKER_BUILDX)/file-repo:$(FILE_REPO_VERSION) -f ./Dockerfile.file-repo .

build/docker/multi-arch/Dockerfile.torrent

@@ -1,24 +1,24 @@
 FROM registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5 as build
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
+
 RUN mkdir -p /root/go/src/yunion.io/x/onecloud
 COPY . /root/go/src/yunion.io/x/onecloud
+RUN cd /root/go/src/yunion.io/x/onecloud && make cmd/torrent
 
-WORKDIR /root/go/src/yunion.io/x/onecloud
-RUN make cmd/torrent
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+MAINTAINER "Zexi Li <zexi.li@icloud.com>"
 
-MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
+# HACK: use v3.14 alpine mirror to upgrade qemu-img
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/main > /etc/apk/repositories
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/community >> /etc/apk/repositories
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates qemu-img && \
+    apk add --no-cache qemu-img && \
     rm -rf /var/cache/apk/*
 
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV TZ Asia/Shanghai
-
+# TAG=20210815.0
 # add executable file torrent
 # make cmd/torrent
 RUN mkdir -p /opt/yunion/bin

build/docker/multi-arch/Dockerfile.torrent.dockerignore

@@ -0,0 +1,2 @@
+_output/alpine-build/_cache
+

cmd/climc/shell/cloudevent/cloudevents.go

@@ -40,4 +40,18 @@ func init() {
 		printList(result, modules.Cloudevents.GetColumns(s))
 		return nil
 	})
+
+	type CloudeventLogsPurgeOptions struct {
+	}
+	R(&CloudeventLogsPurgeOptions{}, "cloud-event-purge", "Purge obsolete cloud event logs", func(s *mcclient.ClientSession, opts *CloudeventLogsPurgeOptions) error {
+		_, err := modules.Cloudevents.PerformClassAction(s, "purge-splitable", nil)
+		if err != nil {
+			return err
+		}
+		_, err = modules.CloudeventLogs.PerformClassAction(s, "purge-splitable", nil)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
 }

cmd/climc/shell/cloudproxy/proxy_agents.go

@@ -28,4 +28,5 @@ func init() {
 	cmd.List(&options.ProxyAgentListOptions{})
 	cmd.Show(&options.ProxyAgentShowOptions{})
 	cmd.Delete(&options.ProxyAgentDeleteOptions{})
+	cmd.Perform("purge-forwards", &options.ProxyAgentPurgeForwardsOptions{})
 }

cmd/climc/shell/compute/cloudaccounts.go

@@ -37,6 +37,7 @@ func init() {
 	cmd.CreateWithKeyword("create-aws", &options.SAWSCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-openstack", &options.SOpenStackCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-huawei", &options.SHuaweiCloudAccountCreateOptions{})
+	cmd.CreateWithKeyword("create-hcso", &options.SHCSOAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-ucloud", &options.SUcloudCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-zstack", &options.SZStackCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-s3", &options.SS3CloudAccountCreateOptions{})
@@ -54,6 +55,7 @@ func init() {
 	cmd.UpdateWithKeyword("update-aws", &options.SAWSCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-openstack", &options.SOpenStackCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-huawei", &options.SHuaweiCloudAccountUpdateOptions{})
+	cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{})
@@ -69,6 +71,7 @@ func init() {
 	cmd.PerformWithKeyword("update-credential-aws", "update-credential", &options.SAWSCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-openstack", "update-credential", &options.SOpenStackCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-huawei", "update-credential", &options.SHuaweiCloudAccountUpdateCredentialOptions{})
+	cmd.PerformWithKeyword("update-credential-hcso", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-ucloud", "update-credential", &options.SUcloudCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-zstack", "update-credential", &options.SZStackCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-s3", "update-credential", &options.SS3CloudAccountUpdateCredentialOptions{})

cmd/climc/shell/compute/hosts.go

@@ -15,10 +15,12 @@
 package compute
 
 import (
+	"context"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -26,6 +28,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
@@ -43,7 +46,7 @@ func init() {
 	cmd.Perform("start", &options.BaseIdOptions{})
 	cmd.Perform("stop", &options.BaseIdOptions{})
 	cmd.Perform("reset", &options.BaseIdOptions{})
-	cmd.Perform("delete", &options.BaseIdOptions{})
+	cmd.BatchDelete(&options.BaseIdsOptions{})
 	cmd.Perform("remove-all-netifs", &options.BaseIdOptions{})
 
 	cmd.BatchPerform("enable", &options.BaseIdsOptions{})
@@ -523,9 +526,36 @@ func init() {
 			return e
 		}
 		i, e := modules.Hosts.GetLoginInfo(s, srvid, nil)
+		privateKey := ""
 		if e != nil {
-			return e
+			if httputils.ErrorCode(e) == 404 {
+				var err error
+				privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+				if err != nil {
+					return errors.Wrap(err, "fetch private key")
+				}
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewString(string(args.ID)), "ID")
+				ret, err := modules.Hosts.Get(s, args.ID, params)
+				if err != nil {
+					return errors.Wrap(err, "get host by ID")
+				}
+				ip, err := ret.GetString("access_ip")
+				if err != nil {
+					return errors.Wrap(err, "get the ip of the host")
+				}
+
+				jsonItem := jsonutils.NewDict()
+				jsonItem.Add(jsonutils.NewString(ip), "ip")
+				jsonItem.Add(jsonutils.NewString("root"), "username")
+				jsonItem.Add(jsonutils.NewString(""), "password")
+				jsonItem.Add(jsonutils.NewInt(22), "port")
+				i = jsonItem
+			} else {
+				return e
+			}
 		}
+
 		host, err := i.GetString("ip")
 		if err != nil {
 			return err
@@ -542,7 +572,7 @@ func init() {
 		if args.Port != 22 {
 			port = args.Port
 		}
-		sshCli, err := ssh.NewClient(host, port, user, passwd, "")
+		sshCli, err := ssh.NewClient(host, port, user, passwd, privateKey)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/compute/server_forward.go

@@ -15,7 +15,11 @@
 package compute
 
 import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
@@ -26,3 +30,63 @@ func init() {
 	cmd.Perform("close-forward", &options.ServerCloseForwardOptions{})
 	cmd.Perform("list-forward", &options.ServerListForwardOptions{})
 }
+
+type forwardInfo struct {
+	ProxyAddr string `json:"proxy_addr"`
+	ProxyPort int    `json:"proxy_port"`
+}
+
+func dump(input jsonutils.JSONObject) (*forwardInfo, error) {
+	ret := new(forwardInfo)
+	if err := input.Unmarshal(ret); err != nil {
+		return nil, errors.Wrap(err, "JSONObject unmarshal failed")
+	}
+
+	if ret.ProxyAddr == "" {
+		return nil, errors.Errorf("proxy_addr is empty")
+	}
+
+	if ret.ProxyPort <= 0 {
+		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyPort)
+	}
+
+	return ret, nil
+}
+
+func openForward(session *mcclient.ClientSession, srvid string) (*forwardInfo, error) {
+	opt := &options.ServerOpenForwardOptions{
+		ServerIdOptions: options.ServerIdOptions{
+			ID: srvid,
+		},
+		Proto: "tcp",
+		Port:  22,
+	}
+
+	params, err := opt.Params()
+	if err != nil {
+		return nil, errors.Wrap(err, "get open forward params")
+	}
+	jsonItem, err := modules.Servers.PerformAction(session, opt.ID, "open-forward", params)
+	if err != nil {
+		return nil, err
+	}
+
+	return dump(jsonItem)
+}
+
+func closeForward(session *mcclient.ClientSession, srvid string, fItem *forwardInfo) (jsonutils.JSONObject, error) {
+	opt := &options.ServerCloseForwardOptions{
+		ServerIdOptions: options.ServerIdOptions{
+			ID: srvid,
+		},
+		Proto:     "tcp",
+		ProxyAddr: fItem.ProxyAddr,
+		ProxyPort: fItem.ProxyPort,
+	}
+
+	params, err := opt.Params()
+	if err != nil {
+		return nil, errors.Wrap(err, "close forward params")
+	}
+	return modules.Servers.PerformAction(session, opt.ID, "close-forward", params)
+}

cmd/climc/shell/compute/servers.go

@@ -15,6 +15,7 @@
 package compute
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -88,6 +89,8 @@ func init() {
 	cmd.Perform("create-eip", &options.ServerCreateEipOptions{})
 	cmd.Perform("make-sshable", &options.ServerMakeSshableOptions{})
 	cmd.Perform("migrate-network", &options.ServerMigrateNetworkOptions{})
+	cmd.Perform("set-sshport", &options.ServerSetSshportOptions{})
+	cmd.Perform("have-agent", &options.ServerHaveAgentOptions{})
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))
@@ -98,6 +101,8 @@ func init() {
 	cmd.Get("make-sshable-cmd", new(options.ServerIdOptions))
 	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
 	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
+	cmd.Get("sshport", new(options.ServerIdOptions))
+
 	cmd.GetProperty(&options.ServerStatusStatisticsOptions{})
 
 	type ServerTaskShowOptions struct {
@@ -793,6 +798,16 @@ func init() {
 			return err
 		}
 
+		eip, err := srv.GetString("eip")
+		if err != nil && err.Error() != "Get: key not found" {
+			return err
+		}
+
+		vpcid, err := srv.GetString("vpc_id")
+		if err != nil {
+			return err
+		}
+
 		address := make([]string, 0)
 		nics, err := srv.GetArray("nics")
 		if err != nil {
@@ -807,13 +822,15 @@ func init() {
 			return fmt.Errorf("Not found ip address from server %s", opts.ID)
 		}
 
+		privateKey := ""
 		params := jsonutils.NewDict()
 		if len(opts.Key) > 0 {
-			privateKey, e := ioutil.ReadFile(opts.Key)
+			key, e := ioutil.ReadFile(opts.Key)
 			if e != nil {
 				return e
 			}
-			params.Add(jsonutils.NewString(string(privateKey)), "private_key")
+			params.Add(jsonutils.NewString(string(key)), "private_key")
+			privateKey = string(key)
 		}
 
 		i, e := modules.Servers.GetLoginInfo(s, srvid, params)
@@ -821,37 +838,88 @@ func init() {
 			return e
 		}
 		passwd, err := i.GetString("password")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.Password != "" {
 			passwd = opts.Password
 		}
 		user, err := i.GetString("username")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.User != "" {
 			user = opts.User
 		}
-
-		host := address[0]
-		if opts.Host != "" {
-			host = opts.Host
-		}
 		port := 22
 		if opts.Port != 22 {
 			port = opts.Port
 		}
 
-		sshCli, err := ssh.NewClient(host, port, user, passwd, "")
-		if err != nil {
-			return err
+		var forwardItem *forwardInfo = nil
+		host := address[0]
+		if opts.Host != "" {
+			host = opts.Host
+		} else {
+			if eip != "" {
+				host = eip
+			} else {
+				if vpcid != "default" {
+					forwardItem, err = openForward(s, srvid)
+					if err != nil {
+						return err
+					}
+					host = forwardItem.ProxyAddr
+					port = forwardItem.ProxyPort
+				}
+			}
 		}
+
+		if opts.UseCloudroot {
+			var err error
+			privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+			if err != nil {
+				return err
+			}
+			passwd = ""
+			user = "cloudroot"
+		}
+
+		var sshCli *ssh.Client
+		err = nil
+		for ; sshCli == nil; sshCli, err = ssh.NewClient(host, port, user, passwd, privateKey) {
+			if err == nil {
+				continue
+			}
+			if opts.Host != "" {
+				return err
+			}
+			if forwardItem != nil {
+				closeForward(s, srvid, forwardItem)
+				return err
+			} else {
+				if vpcid != "default" {
+					forwardItem, e = openForward(s, srvid)
+					if e != nil {
+						return e
+					}
+					host = forwardItem.ProxyAddr
+					port = forwardItem.ProxyPort
+				}
+			}
+		}
+
 		log.Infof("ssh %s:%d", host, port)
 		if err := sshCli.RunTerminal(); err != nil {
+			if forwardItem != nil {
+				closeForward(s, srvid, forwardItem)
+			}
 			return err
 		}
+
+		if forwardItem != nil {
+			closeForward(s, srvid, forwardItem)
+		}
 		return nil
 	})
 

cmd/climc/shell/devtool/script.go

@@ -24,6 +24,7 @@ func init() {
 	cmd := shell.NewResourceCmd(&modules.DevToolScripts).WithKeyword("devtool-script")
 	cmd.List(new(options.ScriptListOptions))
 	cmd.Perform("apply", new(options.ScriptApplyOptions))
+	cmd.Perform("batch-apply", new(options.ScriptBatchApplyOptions))
 	cmd1 := shell.NewResourceCmd(&modules.DevToolScriptApplyRecords).WithKeyword("devtool-script-record")
 	cmd1.List(new(options.ScriptApplyRecordListOptions))
 }

cmd/climc/shell/devtool/serviceurl.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/devtool"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.DevToolServiceUrls).WithKeyword("devtool-serviceurl")
+	cmd.List(new(options.ServiceUrlListOptions))
+	cmd.Create(new(options.ServiceUrlCreateOptions))
+	cmd.Show(new(options.ServiceUrlOptions))
+	cmd.Delete(new(options.ServiceUrlOptions))
+}

cmd/climc/shell/devtool/sshinfo.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/devtool"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.DevToolSshInfos).WithKeyword("devtool-sshinfo")
+	cmd.List(new(options.SshInfoListOptions))
+	cmd.Create(new(options.SshInfoCreateOptions))
+	cmd.Show(new(options.SshInfoOptions))
+	cmd.Delete(new(options.SshInfoOptions))
+}

cmd/climc/shell/events/splitable.go

@@ -24,7 +24,7 @@ import (
 
 func init() {
 	type EventSplitableOptions struct {
-		Service string `help:"service" choices:"compute|identity|image" default:"compute"`
+		Service string `help:"service" choices:"compute|identity|image|log|cloudevent" default:"compute"`
 	}
 	R(&EventSplitableOptions{}, "logs-splitable", "Show splitable info of event table", func(s *mcclient.ClientSession, args *EventSplitableOptions) error {
 		var results jsonutils.JSONObject
@@ -34,6 +34,10 @@ func init() {
 			results, err = modules.IdentityLogs.Get(s, "splitable", nil)
 		case "image":
 			results, err = modules.ImageLogs.Get(s, "splitable", nil)
+		case "log":
+			results, err = modules.Actions.Get(s, "splitable", nil)
+		case "cloudevent":
+			results, err = modules.Cloudevents.Get(s, "splitable", nil)
 		default:
 			results, err = modules.Logs.Get(s, "splitable", nil)
 		}
@@ -58,6 +62,10 @@ func init() {
 			results, err = modules.IdentityLogs.PerformClassAction(s, "purge-splitable", nil)
 		case "image":
 			results, err = modules.ImageLogs.PerformClassAction(s, "purge-splitable", nil)
+		case "log":
+			results, err = modules.Actions.PerformClassAction(s, "purge-splitable", nil)
+		case "cloudevent":
+			results, err = modules.Cloudevents.PerformClassAction(s, "purge-splitable", nil)
 		default:
 			results, err = modules.Logs.PerformClassAction(s, "purge-splitable", nil)
 		}

cmd/climc/shell/image/images.go

@@ -142,13 +142,15 @@ func init() {
 	type ImageListOptions struct {
 		options.BaseListOptions
 
-		IsPublic   string   `help:"filter images public or not(True, False or None)" choices:"true|false"`
-		IsStandard string   `help:"filter images standard or non-standard" choices:"true|false"`
-		Protected  string   `help:"filter images by protected" choices:"true|false"`
-		IsUefi     bool     `help:"list uefi image"`
-		Format     []string `help:"Disk formats"`
-		SubFormats []string `help:"Sub formats"`
-		Name       string   `help:"Name filter"`
+		IsPublic     string   `help:"filter images public or not(True, False or None)" choices:"true|false"`
+		IsStandard   string   `help:"filter images standard or non-standard" choices:"true|false"`
+		Protected    string   `help:"filter images by protected" choices:"true|false"`
+		IsUefi       bool     `help:"list uefi image"`
+		Format       []string `help:"Disk formats"`
+		SubFormats   []string `help:"Sub formats"`
+		Name         string   `help:"Name filter"`
+		OsType       []string `help:"Type of OS filter e.g. 'Windows, Linux, Freebsd, Android, macOS, VMWare'"`
+		Distribution []string `help:"Distribution filter, e.g. 'CentOS, Ubuntu, Debian, Windows'"`
 	}
 	R(&ImageListOptions{}, "image-list", "List images", func(s *mcclient.ClientSession, args *ImageListOptions) error {
 		params, err := args.Params()
@@ -187,6 +189,12 @@ func init() {
 		if len(args.SubFormats) > 0 {
 			params.Add(jsonutils.Marshal(args.SubFormats), "sub_formats")
 		}
+		if len(args.OsType) > 0 {
+			params.Add(jsonutils.NewStringArray(args.OsType), "os_types")
+		}
+		if len(args.Distribution) > 0 {
+			params.Add(jsonutils.NewStringArray(args.Distribution), "distributions")
+		}
 		result, err := modules.Images.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/misc/licenses.go

@@ -15,6 +15,12 @@
 package misc
 
 import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -40,6 +46,40 @@ func init() {
 		return nil
 	})
 
+	type LicenseUploadOptions struct {
+		FILE string
+	}
+
+	R(&LicenseUploadOptions{}, "license-upload", "Upload license", func(s *mcclient.ClientSession, args *LicenseUploadOptions) error {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("license", args.FILE)
+		if err != nil {
+			return err
+		}
+		fh, err := os.Open(args.FILE)
+		if err != nil {
+			return err
+		}
+		defer fh.Close()
+		_, err = io.Copy(fileWriter, fh)
+		if err != nil {
+			return err
+		}
+		contentType := bodyWriter.FormDataContentType()
+		bodyWriter.Close()
+
+		header := http.Header{}
+		header.Set("Content-Type", contentType)
+		lic, err := modules.License.Upload(s, header, bodyBuf)
+		if err != nil {
+			return err
+		}
+
+		printObject(lic)
+		return nil
+	})
+
 	type LicenseShowOptions struct {
 		SERVICE string `help:"service name"  choices:"compute|service_tree"`
 	}

cmd/climc/shell/monitor/monitor_resource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package monitor
 
 import (

cmd/cloudmon/main.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/version"
+
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) {
+	if len(options.AuthURL) == 0 {
+		return nil, errors.Error("empty auth_url")
+	}
+	if len(options.AdminUser) == 0 {
+		return nil, errors.Error("empty admin_user")
+	}
+	if len(options.AdminPassword) == 0 {
+		return nil, errors.Error("empty admin_password")
+	}
+	if len(options.AdminProject) == 0 {
+		return nil, errors.Error("empty admin_project")
+	}
+
+	client := mcclient.NewClient(
+		options.AuthURL,
+		options.Timeout,
+		options.Debug,
+		options.Insecure,
+		options.CertFile,
+		options.KeyFile,
+	)
+
+	token, err := client.AuthenticateWithSource(
+		options.AdminUser,
+		options.AdminPassword,
+		options.AdminDomain,
+		options.AdminProject,
+		options.AdminProjectDomain,
+		mcclient.AuthSourceAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	session := client.NewSession(
+		context.Background(),
+		options.Region,
+		"",
+		options.EndpointType,
+		token,
+		options.ApiVersion)
+
+	return session, nil
+}
+
+func main() {
+	parser, err := options.GetArgumentParser()
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	err = parser.ParseArgs2(os.Args[1:], false, false)
+
+	opts := parser.Options().(*options.CloudMonOptions)
+
+	if opts.Help {
+		fmt.Println(parser.HelpString())
+		os.Exit(0)
+	}
+
+	if opts.Version {
+		fmt.Println(version.GetJsonString())
+		os.Exit(0)
+	}
+
+	if len(opts.Config) == 0 {
+		for _, p := range []string{"./etc", "/etc/yunion"} {
+			confTmp := path.Join(p, "cloudmon.conf")
+			if _, err := os.Stat(confTmp); err == nil {
+				opts.Config = confTmp
+				break
+			}
+		}
+	}
+
+	if len(opts.Config) > 0 {
+		err := parser.ParseFile(opts.Config)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+	}
+
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	parser.SetDefault()
+
+	subcmd := parser.GetSubcommand()
+	subparser := subcmd.GetSubParser()
+	if err != nil {
+		if subparser != nil {
+			fmt.Print(subparser.Usage())
+		} else {
+			fmt.Print(parser.Usage())
+		}
+		showErrorAndExit(err)
+	}
+
+	suboptions := subparser.Options()
+	if opts.SUBCOMMAND == "help" {
+		err = subcmd.Invoke(suboptions)
+	} else {
+		timout := suboptions.(*common.ReportOptions).Timeout
+		endChan := make(chan int, 1)
+		go func() {
+			ticker := time.NewTicker(time.Duration(timout) * time.Second)
+			for {
+				select {
+				case <-ticker.C:
+					log.Errorf("cmd: %s,provider: %v,end due to timeout: %ds", opts.SUBCOMMAND,
+						suboptions.(*common.ReportOptions).Provider, suboptions.(*common.ReportOptions).Timeout)
+					os.Exit(3)
+				case <-endChan:
+				}
+			}
+		}()
+
+		var session *mcclient.ClientSession
+		session, err = newClientSession(opts)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+		err = subcmd.Invoke(session, suboptions)
+		endChan <- 1
+	}
+	if err != nil {
+		showErrorAndExit(err)
+	}
+}

cmd/cryptool/shell/fernet.go

@@ -16,7 +16,6 @@ package shell
 
 import (
 	"fmt"
-	"time"
 
 	"yunion.io/x/pkg/util/timeutils"
 
@@ -110,7 +109,7 @@ func init() {
 		}
 		fmt.Println("primary key hash:", fm.PrimaryKeyHash())
 
-		ret := fm.Decrypt([]byte(args.MSG), time.Hour*-1)
+		ret := fm.Decrypt([]byte(args.MSG))
 		if len(ret) == 0 {
 			return fmt.Errorf("invalid message")
 		}

cmd/hcsocli/main.go

@@ -0,0 +1,147 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	huawei "yunion.io/x/onecloud/pkg/multicloud/hcso"
+	_ "yunion.io/x/onecloud/pkg/multicloud/hcso/shell"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BaseOptions struct {
+	cloudprovider.SHCSOEndpoints
+	Help       bool   `help:"Show help" default:"false"`
+	Debug      bool   `help:"Show debug" default:"false"`
+	AccessKey  string `help:"Access key" default:"$HUAWEI_ACCESS_KEY" metavar:"HUAWEI_ACCESS_KEY"`
+	Secret     string `help:"Secret" default:"$HUAWEI_SECRET" metavar:"HUAWEI_SECRET"`
+	RegionId   string `help:"RegionId" default:"$HUAWEI_REGION" metavar:"HUAWEI_REGION"`
+	ProjectId  string `help:"ProjectId" default:"$HUAWEI_PROJECT" metavar:"HUAWEI_PROJECT"`
+	SUBCOMMAND string `help:"huaweicli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"huaweicli",
+		"Command-line interface to huawei API.",
+		`See "huaweicli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*huawei.SRegion, error) {
+	if len(options.AccessKey) == 0 {
+		return nil, fmt.Errorf("Missing accessKey")
+	}
+
+	if len(options.Secret) == 0 {
+		return nil, fmt.Errorf("Missing secret")
+	}
+
+	cli, err := huawei.NewHuaweiClient(
+		huawei.NewHuaweiClientConfig(
+			options.AccessKey,
+			options.Secret,
+			options.ProjectId,
+			&options.SHCSOEndpoints,
+		).Debug(options.Debug),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	region := cli.GetRegion(options.RegionId)
+	if region == nil {
+		return nil, fmt.Errorf("No such region %s", options.RegionId)
+	}
+
+	return region, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var region *huawei.SRegion
+				region, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(region, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/telegraf-raid-plugin/main.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 const (
@@ -128,6 +129,7 @@ func (c *RaidInfoCollector) CollectReportData() string {
 }
 
 func (c *RaidInfoCollector) Start() {
+	go procutils.WaitZombieLoop(context.TODO())
 	for {
 		c.runMain()
 		time.Sleep(time.Second * 1)

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/coredns/coredns v1.3.0
 	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/creack/pty v1.1.11
 	github.com/dnaeon/go-vcr v1.1.0 // indirect
 	github.com/dnstap/golang-dnstap v0.0.0-20170829151710-2cf77a2b5e11 // indirect
 	github.com/eapache/go-resiliency v1.1.0 // indirect
@@ -54,7 +55,6 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.5.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
-	github.com/go-ini/ini v1.62.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
@@ -77,7 +77,6 @@ require (
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
-	github.com/kr/pty v1.1.5
 	github.com/lestrrat-go/jwx v1.0.2
 	github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae
 	github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect
@@ -113,6 +112,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
+	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect
@@ -146,13 +146,13 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
-	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
+	yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
+	yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c
+	yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824
+	yunion.io/x/sqlchemy v1.0.0
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -2,14 +2,12 @@ bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVc
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc=
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM=
 cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
@@ -32,16 +30,12 @@ github.com/Azure/azure-sdk-for-go v36.1.0+incompatible h1:smHlbChr/JDmsyUqELZXLs
 github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.3 h1:OZEIaBbMdUE/Js+BQKlpO81XlISgipr6yDJ+PSwsgi4=
 github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
 github.com/Azure/go-autorest/autorest v0.9.6 h1:5YWtOnckcudzIw8lPPBcWOnmIFWMtHci1ZWAZulMSx0=
 github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
-github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
-github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk=
 github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
@@ -49,12 +43,10 @@ github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwp
 github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM=
 github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U=
 github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw=
-github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
 github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
 github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
 github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
@@ -180,6 +172,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -241,8 +235,6 @@ github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
-github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
@@ -405,8 +397,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
@@ -586,6 +576,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:nt2877sKfojlHCTOBXbpWjBkuWKritFaGIfgQwbQUls=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:B4+Kq1u5FlULTjFSM707Q6e/cOHFv0z/6QRoxubDIQ8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=
@@ -701,10 +693,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -750,7 +740,6 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -910,11 +899,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a h1:X/ucfLNcYsCfc4GFEX7R7GNWiDPr3vopLdnE4PuU46U=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o=
@@ -924,11 +913,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c h1:/ejlt78/0xod4bdkEThUuYsQ7vLajd48wpL2GMMSaXs=
-yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 h1:3/VcCiO+vr3+zZFLNvre41ARF2xE6NVq6Ue7JPFuJvU=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v1.0.0 h1:fRtWxe1xxMR+NDVKYNuI89m2HGSl6aad2DZ+Bn/mZ4Q=
+yunion.io/x/sqlchemy v1.0.0/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/models/ansibleplaybooks.go

@@ -121,7 +121,7 @@ func (man *SAnsiblePlaybookManager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/ansibleserver/models/ansibleplaybooks_v2.go

@@ -43,7 +43,7 @@ type SAnsiblePlaybookV2 struct {
 
 	Playbook     string    `length:"text" nullable:"false" create:"required" get:"user"`
 	Inventory    string    `length:"text" nullable:"false" create:"required" get:"user"`
-	Requirements string    `length:"test" nullable:"false" create:"optional" get:"user"`
+	Requirements string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Files        string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Output       string    `length:"medium" get:"user"`
 	StartTime    time.Time `list:"user"`
@@ -118,7 +118,7 @@ func (man *SAnsiblePlaybookV2Manager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/apigateway/clientman/authtoken.go

@@ -275,7 +275,7 @@ func (t *SAuthToken) VerifyTotpPasscode(s *mcclient.ClientSession, uid, passcode
 	}
 
 	t.updateRetryCount()
-	return errors.Wrap(httperrors.ErrInvalidCredential, "invalid passcode")
+	return errors.Wrap(httperrors.ErrInputParameter, "invalid passcode")
 }
 
 func SignJWT(t jwt.Token) (string, error) {

pkg/apigateway/handler/auth.go

@@ -185,7 +185,22 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 
 	resp.Add(jsonutils.NewArray(retIdps...), "idps")
 
-	resp.Add(jsonutils.NewString(options.Options.ApiServer), "api_server")
+	// in case api_server changed but not synchronized from Keystone
+	// fetch this option directly from Keystone
+	var apiSrv string
+	commonCfg, err := modules.ServicesV3.GetSpecific(s, "common", "config", nil)
+	if err != nil {
+		log.Errorf("fetch common config error: %s", err)
+	} else {
+		apiSrv, err = commonCfg.GetString("config", "default", "api_server")
+		if err != nil {
+			log.Errorf("fetch api_server fail: %s (cfg: %s)", err, commonCfg)
+		}
+	}
+	if len(apiSrv) == 0 {
+		apiSrv = options.Options.ApiServer
+	}
+	resp.Add(jsonutils.NewString(apiSrv), "api_server")
 
 	return resp, nil
 }
@@ -1183,16 +1198,27 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 		return
 	}
 
-	oldPwd, _ := body.GetString("password_old")
-	newPwd, _ := body.GetString("password_new")
-	confirmPwd, _ := body.GetString("password_confirm")
-	passcode, _ := body.GetString("passcode")
+	input := struct {
+		PasswordOld     string
+		PasswordNew     string
+		PasswordConfirm string
+		Passcode        string
+	}{}
+	body.Unmarshal(&input)
 
-	if newPwd != confirmPwd {
+	if input.PasswordNew != input.PasswordConfirm {
 		httperrors.InputParameterError(ctx, w, "new password mismatch")
 		return
 	}
 
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordOld); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordOld = string(decPasswd)
+	}
+
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordNew); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordNew = string(decPasswd)
+	}
+
 	// 1.验证原密码正确，且idp_driver为空
 	if isIdpUser(user) {
 		httperrors.ForbiddenError(ctx, w, "not support reset user password")
@@ -1200,7 +1226,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	}
 
 	cliIp := netutils2.GetHttpRequestIp(req)
-	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), oldPwd, t.GetDomainName(), "", "", cliIp)
+	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), input.PasswordOld, t.GetDomainName(), "", "", cliIp)
 	if err != nil {
 		switch httperr := err.(type) {
 		case *httputils.JSONClientError:
@@ -1216,7 +1242,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	// 2.如果已开启MFA，验证 随机密码正确
 	if isMfaEnabled(user) {
-		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), passcode)
+		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), input.Passcode)
 		if err != nil {
 			httperrors.InputParameterError(ctx, w, "invalid passcode")
 			return
@@ -1225,7 +1251,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 
 	// 3.重置密码，
 	params := jsonutils.NewDict()
-	params.Set("password", jsonutils.NewString(newPwd))
+	params.Set("password", jsonutils.NewString(input.PasswordNew))
 	_, err = modules.UsersV3.Patch(s, t.GetUserId(), params)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)

pkg/apigateway/handler/auth_totp.go

@@ -260,7 +260,7 @@ func validatePasscodeHandler(ctx context.Context, w http.ResponseWriter, req *ht
 
 	if err != nil {
 		log.Warningf("VerifyTotpPasscode %s", err.Error())
-		httperrors.InvalidCredentialError(ctx, w, "invalid passcode: %v", err)
+		httperrors.InputParameterError(ctx, w, "invalid passcode: %v", err)
 		return
 	}
 

pkg/apigateway/handler/dispatcher.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"time"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
@@ -51,7 +52,17 @@ func (h SHandler) Bind(app *appsrv.Application) {
 	if h.MiddlewareFunc != nil {
 		f = h.MiddlewareFunc(f)
 	}
-	app.AddHandler(h.Method, h.Prefix, f)
+	segs := appsrv.SplitPath(h.Prefix)
+	hi := appsrv.NewHandlerInfo(h.Method, segs, f, nil, "", nil)
+	if h.Method == "GET" {
+		hi.SetProcessTimeoutCallback(func(hdr *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+			if len(r.URL.Query().Get("export_keys")) > 0 {
+				return time.Hour * 2
+			}
+			return -time.Second
+		})
+	}
+	app.AddHandler3(hi)
 }
 
 func NewMethodHandlerFactory(method string, mf appsrv.MiddlewareFunc, prefix string) func(handleFunc, ...string) SHandler {

pkg/apigateway/handler/oidc.go

@@ -100,6 +100,7 @@ func handleOIDCAuth(ctx context.Context, w http.ResponseWriter, req *http.Reques
 	qs.Set("code", jsonutils.NewString(code))
 	qs.Set("state", jsonutils.NewString(auth.State))
 	redirUrl := addQuery(auth.RedirectUri, qs)
+	appsrv.DisableClientCache(w)
 	appsrv.SendRedirect(w, redirUrl)
 }
 

pkg/apihelper/apihelper.go

@@ -101,7 +101,7 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) {
 
 	s := h.adminClientSession(ctx)
 	mss := h.modelSets.Copy()
-	r, err := SyncModelSets(mss, s, h.opts.ListBatchSize)
+	r, err := SyncModelSets(mss, s, h.opts)
 	if err != nil {
 		return false, err
 	}
@@ -118,7 +118,7 @@ func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess
 	if s != nil {
 		token := s.GetToken()
 		expires := token.GetExpires()
-		if time.Now().Add(time.Hour).After(expires) {
+		if time.Now().Add(time.Hour).Before(expires) {
 			return s
 		}
 	}

pkg/apihelper/interface.go

@@ -15,6 +15,8 @@
 package apihelper
 
 import (
+	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	mcclient "yunion.io/x/onecloud/pkg/mcclient"
 	mcclient_modulebase "yunion.io/x/onecloud/pkg/mcclient/modulebase"
@@ -44,7 +46,15 @@ type IModelSetEmulatedIncluder interface {
 	IncludeEmulated() bool
 }
 
-func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int) (r ModelSetsUpdateResult, err error) {
+type IModelSetFilter interface {
+	ModelFilter() []string
+}
+
+type IModelListParam interface {
+	ModelParamFilter() jsonutils.JSONObject
+}
+
+func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
 	mss := mssOld.ModelSetList()
 	mssNews := mssOld.NewEmpty()
 	for i, msNew := range mssNews.ModelSetList() {
@@ -60,10 +70,11 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int)
 			ModelManager:  msNew.ModelManager(),
 			MinUpdatedAt:  minUpdatedAt,
 			ModelSet:      msNew,
-			BatchListSize: batchSize,
+			BatchListSize: opt.ListBatchSize,
 
-			IncludeDetails:  false,
-			IncludeEmulated: includeEmulated,
+			IncludeDetails:       opt.IncludeDetails,
+			IncludeEmulated:      includeEmulated,
+			InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv,
 		})
 		if err != nil {
 			return

pkg/apihelper/options.go

@@ -21,6 +21,8 @@ import (
 type Options struct {
 	common_options.CommonOptions
 
-	SyncInterval  int
-	ListBatchSize int
+	SyncInterval         int
+	ListBatchSize        int
+	IncludeDetails       bool
+	IncludeOtherCloudEnv bool
 }

pkg/apihelper/reflect.go

@@ -38,10 +38,11 @@ type GetModelsOptions struct {
 	ModelManager  mcclient_modulebase.IBaseManager
 	ModelSet      IModelSet
 
-	BatchListSize   int
-	MinUpdatedAt    time.Time
-	IncludeDetails  bool
-	IncludeEmulated bool
+	BatchListSize        int
+	MinUpdatedAt         time.Time
+	IncludeDetails       bool
+	IncludeEmulated      bool
+	InCludeOtherCloudEnv bool
 }
 
 func GetModels(opts *GetModelsOptions) error {
@@ -102,19 +103,28 @@ func GetModels(opts *GetModelsOptions) error {
 
 	listOptions := options.BaseListOptions{
 		Admin:        options.Bool(true),
+		Scope:        "system",
 		Details:      options.Bool(opts.IncludeDetails),
 		ShowEmulated: options.Bool(opts.IncludeEmulated),
 		Filter: []string{
 			minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0
-			"manager_id.isnullorempty()",     // len(manager_id) > 0 is for pubcloud objects
-			"external_id.isnullorempty()",    // len(external_id) > 0 is for pubcloud objects
-			"cloud_env=onpremise",
 		},
 		OrderBy: []string{"updated_at"},
 		Order:   "asc",
 		Limit:   options.Int(opts.BatchListSize),
 		Offset:  options.Int(0),
 	}
+	if !opts.InCludeOtherCloudEnv {
+		listOptions.Filter = append(listOptions.Filter,
+			"manager_id.isnullorempty()",  // len(manager_id) > 0 is for pubcloud objects
+			"external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
+			"cloud_env=onpremise",
+		)
+	}
+	if inter, ok := opts.ModelSet.(IModelSetFilter); ok {
+		filter := inter.ModelFilter()
+		listOptions.Filter = append(listOptions.Filter, filter...)
+	}
 	if !minUpdatedAt.Equal(PseudoZeroTime) {
 		// Only fetching pending deletes when we are doing incremental fetch
 		listOptions.PendingDeleteAll = options.Bool(true)
@@ -124,6 +134,10 @@ func GetModels(opts *GetModelsOptions) error {
 	if err != nil {
 		return fmt.Errorf("%s: making list params: %s", manKeyPlural, err)
 	}
+	if inter, ok := opts.ModelSet.(IModelListParam); ok {
+		filter := inter.ModelParamFilter()
+		params.Update(filter)
+	}
 	//XXX
 	//params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL))
 

pkg/apis/compute/api.go

@@ -396,6 +396,10 @@ type ServerCreateInput struct {
 	// emulate: BIOS, UEFI
 	Bios string `json:"bios"`
 
+	// Machine类型
+	// emulate: pc, q35
+	Machine string `json:"machine"`
+
 	// 启动顺序
 	// c: cdrome
 	// d: disk

pkg/apis/compute/baremetal_const.go

@@ -29,12 +29,13 @@ const (
 
 	DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE
 
-	DISK_DRIVER_MEGARAID   = "MegaRaid"
-	DISK_DRIVER_LINUX      = "Linux"
-	DISK_DRIVER_HPSARAID   = "HPSARaid"
-	DISK_DRIVER_MPT2SAS    = "Mpt2SAS"
-	DISK_DRIVER_MARVELRAID = "MarvelRaid"
-	DISK_DRIVER_PCIE       = "PCIE"
+	DISK_DRIVER_MEGARAID    = "MegaRaid"
+	DISK_DRIVER_LINUX       = "Linux"
+	DISK_DRIVER_HPSARAID    = "HPSARaid"
+	DISK_DRIVER_MPT2SAS     = "Mpt2SAS"
+	DISK_DRIVER_MARVELRAID  = "MarvelRaid"
+	DISK_DRIVER_ADAPTECRAID = "AdaptecRaid"
+	DISK_DRIVER_PCIE        = "PCIE"
 
 	HDD_DISK_SPEC_TYPE = "HDD"
 	SSD_DISK_SPEC_TYPE = "SSD"
@@ -60,6 +61,7 @@ var (
 		DISK_DRIVER_HPSARAID,
 		DISK_DRIVER_MPT2SAS,
 		DISK_DRIVER_MARVELRAID,
+		DISK_DRIVER_ADAPTECRAID,
 	)
 
 	DISK_DRIVERS = sets.NewString(

pkg/apis/compute/cloudaccount_const.go

@@ -44,6 +44,7 @@ const (
 	CLOUD_PROVIDER_AZURE     = "Azure"
 	CLOUD_PROVIDER_AWS       = "Aws"
 	CLOUD_PROVIDER_HUAWEI    = "Huawei"
+	CLOUD_PROVIDER_HCSO      = "HCSO"
 	CLOUD_PROVIDER_OPENSTACK = "OpenStack"
 	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
 	CLOUD_PROVIDER_ZSTACK    = "ZStack"
@@ -94,7 +95,7 @@ const (
 var (
 	CLOUD_PROVIDER_VALID_STATUS        = []string{CLOUD_PROVIDER_CONNECTED}
 	CLOUD_PROVIDER_VALID_HEALTH_STATUS = []string{CLOUD_PROVIDER_HEALTH_NORMAL, CLOUD_PROVIDER_HEALTH_NO_PERMISSION}
-	PRIVATE_CLOUD_PROVIDERS            = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA}
+	PRIVATE_CLOUD_PROVIDERS            = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA, CLOUD_PROVIDER_HCSO}
 
 	CLOUD_PROVIDERS = []string{
 		CLOUD_PROVIDER_ONECLOUD,
@@ -105,6 +106,7 @@ var (
 		CLOUD_PROVIDER_AZURE,
 		CLOUD_PROVIDER_AWS,
 		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_HCSO,
 		CLOUD_PROVIDER_OPENSTACK,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_ZSTACK,
@@ -140,6 +142,9 @@ var (
 		CLOUD_PROVIDER_HUAWEI: []string{
 			HOST_TYPE_HUAWEI,
 		},
+		CLOUD_PROVIDER_HCSO: {
+			HOST_TYPE_HCSO,
+		},
 		CLOUD_PROVIDER_OPENSTACK: []string{
 			HOST_TYPE_OPENSTACK,
 		},

pkg/apis/compute/dnszone.go

@@ -125,7 +125,7 @@ type DnsZoneDetails struct {
 	// 关联vpc数量
 	VpcCount int `json:"vpc_count"`
 	// Cache info
-	CloudCaches []DnsZoneCacheDetails `json:"cloud_caches"`
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type DnsZoneListInput struct {
@@ -141,7 +141,8 @@ type DnsZoneListInput struct {
 	ZoneType string `json:"zone_type"`
 
 	// Filter dns zone By vpc
-	VpcId string `json:"vpc_id"`
+	VpcId     string `json:"vpc_id"`
+	WithCache bool   `json:"with_cache"`
 }
 
 type DnsZoneSyncStatusInput struct {

pkg/apis/compute/filesystem.go

@@ -36,6 +36,9 @@ const (
 	// 删除中
 	NAS_STATUS_DELETING      = "deleting"
 	NAS_STATUS_DELETE_FAILED = "delete_failed"
+
+	NAS_UPDATE_TAGS        = "update_tags"
+	NAS_UPDATE_TAGS_FAILED = "update_tags_fail"
 )
 
 type FileSystemListInput struct {
@@ -104,3 +107,8 @@ type FileSystemDetails struct {
 	Network string
 	Zone    string
 }
+
+type FileSystemRemoteUpdateInput struct {
+	// 是否覆盖替换所有标签
+	ReplaceTags *bool `json:"replace_tags" help:"replace all remote tags"`
+}

pkg/apis/compute/guest_const.go

@@ -159,6 +159,7 @@ const (
 	HYPERVISOR_AZURE     = "azure"
 	HYPERVISOR_AWS       = "aws"
 	HYPERVISOR_HUAWEI    = "huawei"
+	HYPERVISOR_HCSO      = "hcso"
 	HYPERVISOR_OPENSTACK = "openstack"
 	HYPERVISOR_UCLOUD    = "ucloud"
 	HYPERVISOR_ZSTACK    = "zstack"
@@ -175,6 +176,11 @@ const (
 	CPU_MODE_HOST = "host"
 )
 
+const (
+	VM_MACHINE_TYPE_PC  = "pc"
+	VM_MACHINE_TYPE_Q35 = "q35"
+)
+
 var VM_RUNNING_STATUS = []string{VM_START_START, VM_STARTING, VM_RUNNING, VM_BLOCK_STREAM, VM_BLOCK_STREAM_FAIL}
 var VM_CREATING_STATUS = []string{VM_CREATE_NETWORK, VM_CREATE_DISK, VM_START_DEPLOY, VM_DEPLOYING}
 
@@ -189,6 +195,7 @@ var HYPERVISORS = []string{
 	HYPERVISOR_AWS,
 	HYPERVISOR_QCLOUD,
 	HYPERVISOR_HUAWEI,
+	HYPERVISOR_HCSO,
 	HYPERVISOR_OPENSTACK,
 	HYPERVISOR_UCLOUD,
 	HYPERVISOR_ZSTACK,
@@ -219,6 +226,7 @@ var PRIVATE_CLOUD_HYPERVISORS = []string{
 	HYPERVISOR_ZSTACK,
 	HYPERVISOR_OPENSTACK,
 	HYPERVISOR_APSARA,
+	HYPERVISOR_HCSO,
 }
 
 // var HYPERVISORS = []string{HYPERVISOR_ALIYUN}
@@ -234,6 +242,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{
 	HYPERVISOR_AWS:       HOST_TYPE_AWS,
 	HYPERVISOR_QCLOUD:    HOST_TYPE_QCLOUD,
 	HYPERVISOR_HUAWEI:    HOST_TYPE_HUAWEI,
+	HYPERVISOR_HCSO:      HOST_TYPE_HCSO,
 	HYPERVISOR_OPENSTACK: HOST_TYPE_OPENSTACK,
 	HYPERVISOR_UCLOUD:    HOST_TYPE_UCLOUD,
 	HYPERVISOR_ZSTACK:    HOST_TYPE_ZSTACK,
@@ -253,6 +262,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 	HOST_TYPE_AWS:        HYPERVISOR_AWS,
 	HOST_TYPE_QCLOUD:     HYPERVISOR_QCLOUD,
 	HOST_TYPE_HUAWEI:     HYPERVISOR_HUAWEI,
+	HOST_TYPE_HCSO:       HYPERVISOR_HCSO,
 	HOST_TYPE_OPENSTACK:  HYPERVISOR_OPENSTACK,
 	HOST_TYPE_UCLOUD:     HYPERVISOR_UCLOUD,
 	HOST_TYPE_ZSTACK:     HYPERVISOR_ZSTACK,
@@ -288,3 +298,7 @@ func Hypervisors2HostTypes(hypervisors []string) []string {
 	}
 	return hostTypes
 }
+
+// windows allow a maximal length of 15
+// http://support.microsoft.com/kb/909264
+const MAX_WINDOWS_COMPUTER_NAME_LENGTH = 15

pkg/apis/compute/guest_metadata.go

@@ -4,6 +4,8 @@ const (
 	MIRROR_JOB        = "__mirror_job_status"
 	MIRROR_JOB_READY  = "ready"
 	MIRROR_JOB_FAILED = "failed"
+
+	SSH_PORT = "__ssh_port"
 )
 
 const BASE_INSTANCE_SNAPSHOT_ID = "__base_instance_snapshot_id"

pkg/apis/compute/guest_sshable.go

@@ -44,10 +44,18 @@ type GuestSshableOutput struct {
 	MethodTried []GuestSshableMethodData
 }
 
+type GuestHaveAgentInput struct {
+}
+
+type GuestHaveAgentOutput struct {
+	Have bool
+}
+
 type GuestMakeSshableInput struct {
 	User       string
 	PrivateKey string
 	Password   string
+	Port       int
 }
 
 type GuestMakeSshableOutput struct {
@@ -57,3 +65,11 @@ type GuestMakeSshableOutput struct {
 type GuestMakeSshableCmdOutput struct {
 	ShellCmd string
 }
+
+type GuestSetSshportInput struct {
+	Port int
+}
+
+type GuestSshportOutput struct {
+	Port int
+}

pkg/apis/compute/host_const.go

@@ -29,6 +29,7 @@ const (
 	HOST_TYPE_QCLOUD    = "qcloud"
 	HOST_TYPE_AZURE     = "azure"
 	HOST_TYPE_HUAWEI    = "huawei"
+	HOST_TYPE_HCSO      = "hcso"
 	HOST_TYPE_OPENSTACK = "openstack"
 	HOST_TYPE_UCLOUD    = "ucloud"
 	HOST_TYPE_ZSTACK    = "zstack"
@@ -109,6 +110,7 @@ var HOST_TYPES = []string{
 	HOST_TYPE_AWS,
 	HOST_TYPE_QCLOUD,
 	HOST_TYPE_HUAWEI,
+	HOST_TYPE_HCSO,
 	HOST_TYPE_OPENSTACK,
 	HOST_TYPE_UCLOUD,
 	HOST_TYPE_ZSTACK,

pkg/apis/compute/hoststorage.go

@@ -16,6 +16,8 @@ package compute
 
 import (
 	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
 )
 
 type HoststorageDetails struct {
@@ -58,3 +60,10 @@ type HoststorageListInput struct {
 
 	StorageFilterListInput
 }
+
+type HostStorageCreateInput struct {
+	apis.JoinResourceBaseCreateInput
+	StorageId  string `json:"storage_id"`
+	HostId     string `json:"host_id"`
+	MountPoint string `json:"mount_point"`
+}

pkg/apis/compute/network.go

@@ -126,6 +126,8 @@ type NetworkListInput struct {
 
 	// filter by BGP types
 	BgpType []string `json:"bgp_type"`
+
+	HostType string `json:"host_type"`
 }
 
 type NetworkResourceInfoBase struct {
@@ -207,22 +209,11 @@ type NetworkCreateInput struct {
 	BgpType string `json:"bgp_type"`
 }
 
-type NetworkDetails struct {
-	apis.SharableVirtualResourceDetails
-	WireResourceInfo
-
-	SNetwork
-
-	// 是否是内网
-	Exit bool `json:"exit"`
-	// 端口数量
-	Ports int `json:"ports"`
-	// 已使用端口数量
-	PortsUsed int `json:"ports_used"`
-	// 网卡数量
+type SNetworkNics struct {
+	// 虚拟机网卡数量
 	Vnics int `json:"vnics"`
 	// 裸金属网卡数量
-	BmVnics int `json:"bm_nics"`
+	BmVnics int `json:"bm_vnics"`
 	// 负载均衡网卡数量
 	LbVnics int `json:"lb_vnics"`
 	// 浮动Ip网卡数量
@@ -231,6 +222,48 @@ type NetworkDetails struct {
 	// 预留IP数量
 	ReserveVnics int `json:"reserve_vnics"`
 
+	// RDS网卡数量
+	RdsVnics int `json:"rds_vnics"`
+	// NAT网关网卡数量
+	NatVnics      int `json:"nat_vnics"`
+	BmReusedVnics int `json:"bm_reused_vnics"`
+
+	// 弹性网卡数量
+	NetworkinterfaceVnics int `json:"networkinterface_vnics"`
+
+	// 已使用端口数量
+	PortsUsed int `json:"ports_used"`
+
+	Total int `json:"total"`
+}
+
+func (self *SNetworkNics) SumTotal() {
+	self.Total = self.Vnics +
+		self.BmVnics +
+		self.LbVnics +
+		self.LbVnics +
+		self.EipVnics +
+		self.GroupVnics +
+		self.ReserveVnics +
+		self.RdsVnics +
+		self.NetworkinterfaceVnics +
+		self.NatVnics -
+		self.BmReusedVnics
+	self.PortsUsed = self.Total
+}
+
+type NetworkDetails struct {
+	apis.SharableVirtualResourceDetails
+	WireResourceInfo
+
+	SNetwork
+	SNetworkNics
+
+	// 是否是内网
+	Exit bool `json:"exit"`
+	// 端口数量
+	Ports int `json:"ports"`
+
 	// 路由信息
 	Routes    [][]string                 `json:"routes"`
 	Schedtags []SchedtagShortDescDetails `json:"schedtags"`

pkg/apis/compute/network_const.go

@@ -59,6 +59,7 @@ var (
 
 	REGIONAL_NETWORK_PROVIDERS = []string{
 		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_HCSO,
 		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,

pkg/apis/compute/routetable_routesets.go

@@ -70,4 +70,7 @@ type RouteTableRouteSetListInput struct {
 	apis.StatusStandaloneResourceListInput
 	apis.ExternalizedResourceBaseListInput
 	RouteTableFilterList
+	Type        string `json:"type"`
+	NextHopType string `json:"next_hop_type"`
+	NextHopId   string `json:"next_hop_id"`
 }

pkg/apis/compute/secgroup.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"fmt"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
@@ -181,6 +182,7 @@ type SecgroupListInput struct {
 	RegionalFilterListInput
 
 	ManagedResourceListInput
+	WithCache bool `json:"witch_cache"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -252,6 +254,8 @@ type SecgroupDetails struct {
 	InRules []SecgroupRuleDetails `json:"in_rules"`
 	// 出方向规则信息
 	OutRules []SecgroupRuleDetails `json:"out_rules"`
+
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/storage_const.go

@@ -68,6 +68,7 @@ const (
 	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
 	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
 	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
+	STORAGE_HUAWEI_ESSD  = "ESSD"  // 急速型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/compute/wire.go

@@ -90,7 +90,8 @@ type WireListInput struct {
 
 	HostResourceInput
 
-	Bandwidth *int `json:"bandwidth"`
+	Bandwidth *int   `json:"bandwidth"`
+	HostType  string `json:"host_type"`
 }
 
 type WireMergeInput struct {

pkg/apis/const.go

@@ -74,10 +74,20 @@ const (
 	OS_ARCH_AARCH64 = "aarch64"
 )
 
-func IsARM(osArch string) bool {
-	return utils.IsInStringArray(osArch, []string{
+var (
+	ARCH_X86 = []string{
+		OS_ARCH_X86,
+		OS_ARCH_I386,
+		OS_ARCH_X86_32,
+		OS_ARCH_X86_64,
+	}
+	ARCH_ARM = []string{
 		OS_ARCH_ARM,
 		OS_ARCH_AARCH32,
 		OS_ARCH_AARCH64,
-	})
+	}
+)
+
+func IsARM(osArch string) bool {
+	return utils.IsInStringArray(osArch, ARCH_ARM)
 }

pkg/apis/devtool/script.go

@@ -29,6 +29,21 @@ type ScriptApplyOutput struct {
 	ScriptApplyId string
 }
 
+type ScriptBatchApplyInput struct {
+	ServerIds []string
+}
+
+type ScriptBatchApplyOutput struct {
+	Results []ScriptBatchApplyResult
+}
+
+type ScriptBatchApplyResult struct {
+	ServerId      string
+	Succeed       bool
+	Reason        string
+	ScriptApplyId string
+}
+
 type ScriptApplyRecoredListInput struct {
 	apis.StatusStandaloneResourceListInput
 	// description: Id of Script
@@ -72,3 +87,12 @@ type SApplyInfo struct {
 	ServerId string
 	TryTimes int
 }
+
+type DevtoolManagerServiceUrlInput struct {
+	ServiceName string
+	ServerId    string
+}
+
+type DevtoolManagerServiceUrlOutput struct {
+	ServiceUrl string
+}

pkg/apis/devtool/serviceurl.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+const (
+	SERVICEURL_STATUS_CREATING      = "creating"
+	SERVICEURL_STATUS_READY         = "ready"
+	SERVICEURL_STATUS_DELETING      = "deleting"
+	SERVICEURL_STATUS_CREATE_FAILED = "create_failed"
+	SERVICEURL_STATUS_DELETE_FAILED = "delete_failed"
+)

pkg/apis/devtool/sshinfo.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+const (
+	SSHINFO_STATUS_CREATING      = "creating"
+	SSHINFO_STATUS_READY         = "ready"
+	SSHINFO_STATUS_DELETING      = "deleting"
+	SSHINFO_STATUS_CREATE_FAILED = "create_failed"
+	SSHINFO_STATUS_DELETE_FAILED = "delete_failed"
+)

pkg/apis/identity/consts.go

@@ -109,6 +109,7 @@ var (
 			"customized_private_prefixes",
 			"global_http_proxy",
 			"global_https_proxy",
+			"ignore_nonrunning_guests",
 		},
 	}
 

pkg/apis/image/consts.go

@@ -41,6 +41,9 @@ const (
 	LocalFilePrefix = "file://"
 	S3Prefix        = "s3://"
 
+	IMAGE_STORAGE_DRIVER_LOCAL = "local"
+	IMAGE_STORAGE_DRIVER_S3    = "s3"
+
 	// image properties
 	IMAGE_OS_ARCH             = "os_arch"
 	IMAGE_OS_DISTRO           = "os_distribution"

pkg/apis/image/guestimage.go

@@ -26,8 +26,6 @@ type GuestImageDetails struct {
 	apis.SharableVirtualResourceDetails
 	SGuestImage
 
-	ImageIds []string `json:"image_ids"`
-
 	//Status     string               `json:"status"`
 	Size       int64          `json:"size"`
 	MinRamMb   int32          `json:"min_ram_mb"`

pkg/apis/image/image.go

@@ -43,6 +43,12 @@ type ImageListInput struct {
 
 	// 是否为数据盘
 	IsData *bool `json:"is_data"`
+
+	// 操作系统类型，可能值为: Linux, Windows, FreeBSD 等
+	OsTypes []string `json:"os_types"`
+
+	// 发行版本，可能值为: CentOS, Ubuntu, Debian, ArchLinux,  OpenEuler 等
+	Distributions []string `json:"distributions"`
 }
 
 type GuestImageListInput struct {

pkg/apis/list.go

@@ -195,10 +195,13 @@ type STag struct {
 }
 
 type MetadataResourceListInput struct {
-	// 通过标签过滤
+	// 通过标签过滤（包含这些标签）
 	Tags []STag `json:"tags"`
 
-	// 通过标签过滤
+	// 通过标签过滤（不包含这些标签）
+	NoTags []STag `json:"no_tags"`
+
+	// 通过标签排序
 	OrderByTag string `json:"order_by_tag"`
 
 	// deprecated

pkg/apis/monitor/alertquery_const.go

@@ -1,6 +1,28 @@
 package monitor
 
 var (
+	UnionTags = []string{
+		"host",
+		"host_id",
+		"id",
+		"ips",
+		"name",
+		"zone",
+		"zone_id",
+		"zone_ext_id",
+		"os_type",
+		"status",
+		"cloudregion",
+		"cloudregion_id",
+		"region_ext_id",
+		"tenant",
+		"tenant_id",
+		"brand",
+		"vm_scaling_group_id",
+		"domain_id",
+		"project_domain",
+	}
+
 	ServerTags = map[string]string{
 		"host":             "host",
 		"host_id":          "host_id",

pkg/apis/monitor/metric.go

@@ -2,6 +2,8 @@ package monitor
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const EXT_PREFIX = "ext"
+
 const (
 	METRIC_RES_TYPE_GUEST        = "guest"
 	METRIC_RES_TYPE_HOST         = "host"
@@ -9,10 +11,15 @@ const (
 	METRIC_RES_TYPE_OSS          = "oss"
 	METRIC_RES_TYPE_RDS          = "rds"
 	METRIC_RES_TYPE_CLOUDACCOUNT = "cloudaccount"
+	METRIC_RES_TYPE_AGENT        = "agent"
 	METRIC_RES_TYPE_TENANT       = "tenant"
 	METRIC_RES_TYPE_DOMAIN       = "domain"
 	METRIC_RES_TYPE_STORAGE      = "storage"
-	METRIC_RES_TYPE_JENKINS      = "jenkins"
+	//ext is prefix！
+	METRIC_RES_TYPE_JENKINS      = "ext_jenkins"
+	METRIC_RES_TYPE_EXT_MYSQL    = "ext_mysql"
+	METRIC_RES_TYPE_EXT_REDIS    = "ext_redis"
+	METRIC_RES_TYPE_EXT_RABBITMQ = "ext_rabbitmq"
 
 	METRIC_UNIT_PERCENT = "%"
 	METRIC_UNIT_BPS     = "bps"
@@ -21,6 +28,7 @@ const (
 	METRIC_UNIT_CPS     = "cps"
 	METRIC_UNIT_COUNT   = "count"
 	METRIC_UNIT_MS      = "ms"
+	METRIC_UNIT_SEC     = "s"
 	METRIC_UNIT_BYTE    = "byte"
 	METRIC_UNIT_RMB     = "RMB"
 	METRIC_UNIT_MB      = "Mb"
@@ -34,8 +42,9 @@ var (
 		METRIC_RES_TYPE_RDS, METRIC_RES_TYPE_CLOUDACCOUNT}
 	MetricUnit = []string{METRIC_UNIT_PERCENT, METRIC_UNIT_BPS, METRIC_UNIT_MBPS, METRIC_UNIT_BYTEPS, "count/s",
 		METRIC_UNIT_COUNT, METRIC_UNIT_MS, METRIC_UNIT_BYTE, METRIC_UNIT_RMB}
-	ResTypeScoreMap = map[string]int{
+	ResTypeScoreMap = map[string]float64{
 		METRIC_RES_TYPE_GUEST:        1,
+		METRIC_RES_TYPE_AGENT:        1.1,
 		METRIC_RES_TYPE_HOST:         2,
 		METRIC_RES_TYPE_OSS:          3,
 		METRIC_RES_TYPE_RDS:          4,

pkg/apis/monitor/monitor_resource.go

@@ -33,6 +33,9 @@ type MonitorResourceListInput struct {
 type MonitorResourceDetails struct {
 	apis.VirtualResourceDetails
 	compute.CloudregionResourceInfo
+	compute.CloudaccountResourceInfo
+	Ips      string `json:"ips"`
+	AccessIp string `json:"access_ip"`
 
 	AttachAlertCount int64 `json:"attach_alert_count"`
 }

pkg/apis/monitor/unifiedmonitor_const.go

@@ -20,10 +20,12 @@ var (
 		METRIC_RES_TYPE_OSS:          "oss_name",
 		METRIC_RES_TYPE_CLOUDACCOUNT: "cloudaccount_name",
 		METRIC_RES_TYPE_STORAGE:      "storage_name",
+		METRIC_RES_TYPE_AGENT:        "vm_name",
 	}
 	MEASUREMENT_TAG_ID = map[string]string{
 		METRIC_RES_TYPE_HOST:         "host_id",
 		METRIC_RES_TYPE_GUEST:        "vm_id",
+		METRIC_RES_TYPE_AGENT:        "vm_id",
 		METRIC_RES_TYPE_REDIS:        "redis_id",
 		METRIC_RES_TYPE_RDS:          "rds_id",
 		METRIC_RES_TYPE_OSS:          "oss_id",
@@ -53,9 +55,11 @@ type MetricFunc struct {
 }
 
 type MetricInputQuery struct {
-	From  string `json:"from"`
-	To    string `json:"to"`
-	Scope string `json:"scope"`
+	From    string `json:"from"`
+	To      string `json:"to"`
+	Scope   string `json:"scope"`
+	Slimit  string `json:"slimit"`
+	Soffset string `json:"soffset"`
 	//default group by
 	Unit        bool          `json:"unit"`
 	Interval    string        `json:"interval"`

pkg/apis/output.go

@@ -40,7 +40,7 @@ type ModelBaseDetails struct {
 	CanUpdate bool `json:"can_update"`
 
 	// 资源不能更新的原因
-	UpdateFailReason string `json:"update_fail_reason"`
+	UpdateFailReason httperrors.Error `json:"update_fail_reason"`
 }
 
 type ModelBaseShortDescDetail struct {

pkg/apis/webconsole/consts.go

@@ -31,4 +31,5 @@ const (
 	CTYUN     = "ctyun"
 	HUAWEI    = "huawei"
 	APSARA    = "apsara"
+	JDCLOUD   = "jdcloud"
 )

pkg/appsrv/appsrv.go

@@ -65,6 +65,7 @@ type Application struct {
 	isExiting       bool
 	idleConnsClosed chan struct{}
 	httpServer      *http.Server
+	slaveHttpServer *http.Server
 }
 
 const (
@@ -464,15 +465,16 @@ func (app *Application) ListenAndServeWithoutCleanup(addr, certFile, keyFile str
 }
 
 func (app *Application) ListenAndServeTLSWithCleanup2(addr string, certFile, keyFile string, onStop func(), isMaster bool) {
+	httpSrv := app.initServer(addr)
 	if isMaster {
 		app.addDefaultHandlers()
 		AddPProfHandler(app)
-	}
-	app.httpServer = app.initServer(addr)
-	if isMaster {
+		app.httpServer = httpSrv
 		app.registerCleanShutdown(app.httpServer, onStop)
+	} else {
+		app.slaveHttpServer = httpSrv
 	}
-	app.listenAndServeInternal(app.httpServer, certFile, keyFile)
+	app.listenAndServeInternal(httpSrv, certFile, keyFile)
 	if isMaster {
 		app.waitCleanShutdown()
 	}

pkg/appsrv/handlerinfo.go

@@ -83,6 +83,10 @@ func (this *SHandlerInfo) GetTags() map[string]string {
 	return this.tags
 }
 
+func NewHandlerInfo(method string, path []string, handler func(context.Context, http.ResponseWriter, *http.Request), metadata map[string]interface{}, name string, tags map[string]string) *SHandlerInfo {
+	return newHandlerInfo(method, path, handler, metadata, name, tags)
+}
+
 func newHandlerInfo(method string, path []string, handler func(context.Context, http.ResponseWriter, *http.Request), metadata map[string]interface{}, name string, tags map[string]string) *SHandlerInfo {
 	hand := SHandlerInfo{method: method, path: path,
 		handler:  handler,

pkg/baremetal/manager.go

@@ -185,6 +185,7 @@ func (m *SBaremetalManager) CleanBaremetal(bmId string) {
 func (m *SBaremetalManager) updateBaremetal(session *mcclient.ClientSession, bmId string) (jsonutils.JSONObject, error) {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.JSONTrue, "is_baremetal")
+	params.Add(jsonutils.JSONTrue, "not_sync_config")
 	obj, err := modules.Hosts.Put(session, bmId, params)
 	if err != nil {
 		return nil, err
@@ -1102,10 +1103,12 @@ LABEL start
 		bootmode := api.BOOT_MODE_PXE
 		if !isTftp {
 			adminNic := b.GetAdminNic()
+			var mac string
 			var addr string
 			var mask string
 			var gateway string
 			if adminNic != nil {
+				mac = adminNic.GetMac().String()
 				addr = adminNic.IpAddr
 				mask = adminNic.GetNetMask()
 				gateway = adminNic.Gateway
@@ -1119,6 +1122,7 @@ LABEL start
 				}
 			}
 			serverIP, _ := b.manager.Agent.GetDHCPServerIP()
+			args = append(args, fmt.Sprintf("mac=%s", mac))
 			args = append(args, fmt.Sprintf("dest=%s", serverIP))
 			args = append(args, fmt.Sprintf("gateway=%s", gateway))
 			args = append(args, fmt.Sprintf("addr=%s", addr))
@@ -1325,6 +1329,11 @@ func (b *SBaremetalInstance) GetAccessIp() string {
 	return accessIp
 }
 
+func (b *SBaremetalInstance) GetAccessMac() string {
+	accessMac, _ := b.desc.GetString("access_mac")
+	return accessMac
+}
+
 func (b *SBaremetalInstance) GetServer() baremetaltypes.IBaremetalServer {
 	b.serverLock.Lock()
 	defer b.serverLock.Unlock()
@@ -2440,9 +2449,7 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) error {
 		}
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
 		return err
 	}
@@ -2450,6 +2457,7 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) error {
 	for tried := 0; !tool.IsAllDisksReady() && tried < maxTries; tried++ {
 		time.Sleep(5 * time.Second)
 		tool.RetrieveDiskInfo()
+		log.Warningf("disktool not ready string: %s", tool.DebugString())
 	}
 
 	if !tool.IsAllDisksReady() {
@@ -2520,11 +2528,9 @@ func (s *SBaremetalServer) DoPartitionDisk(term *ssh.Client) ([]*disktool.Partit
 		return nil, errors.Wrapf(err, "CalculateLayout")
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
-		return nil, errors.Wrapf(err, "RetrieveDiskInfo")
+		return nil, errors.Wrapf(err, "NewSSHPartitionTool")
 	}
 
 	disks, _ := s.desc.GetArray("disks")
@@ -2597,9 +2603,7 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 		return nil, err
 	}
 
-	tool := disktool.NewSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
-	err = tool.RetrieveDiskInfo()
+	tool, err := disktool.NewSSHPartitionTool(term, layouts)
 	if err != nil {
 		return nil, err
 	}
@@ -2655,7 +2659,13 @@ func (s *SBaremetalServer) DoRebuildRootDisk(term *ssh.Client) ([]*disktool.Part
 
 func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {
 	disks, _ := s.desc.GetArray("disks")
+
+	// calculate root partitions count
 	rootPartsCnt := len(parts) - len(disks) + 1
+	if len(parts) < len(disks) {
+		// HACK: rebuild root disk
+		rootPartsCnt = len(parts)
+	}
 	rootParts := parts[0:rootPartsCnt]
 	dataParts := parts[rootPartsCnt:]
 	idx := 0
@@ -2714,7 +2724,58 @@ func (s *SBaremetalServer) deployFs(term *ssh.Client, deployInfo *deployapi.Depl
 	if strings.ToLower(rootfs.GetOs()) == "windows" {
 		return nil, fmt.Errorf("Unsupported OS: %s", rootfs.GetOs())
 	}
-	return guestfs.DeployGuestFs(rootfs, s.desc, deployInfo)
+	desc, err := deployapi.GuestDescToDeployDesc(s.desc)
+	if err != nil {
+		return nil, errors.Wrap(err, "To deploy desc fail")
+	}
+	desc, err = s.reIndexDescNics(term, desc)
+	if err != nil {
+		return nil, errors.Wrap(err, "reIndexDescNics")
+	}
+
+	return guestfs.DeployGuestFs(rootfs, desc, deployInfo)
+}
+
+func (s *SBaremetalServer) reIndexDescNics(term *ssh.Client, desc *deployapi.GuestDesc) (*deployapi.GuestDesc, error) {
+	rNics, err := tasks.GetNicsInfo(term)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Get server %s remote nics", s.GetName())
+	}
+	findRemoteNic := func(mac string) (int, *types.SNicDevInfo) {
+		for idx, nic := range rNics {
+			if nic.Mac.String() == mac {
+				return idx, rNics[idx]
+			}
+		}
+		return -1, nil
+	}
+	reIndexNics := func(nics []*deployapi.Nic) ([]*deployapi.Nic, error) {
+		for idx, nic := range nics {
+			if nic.GetNicType() == api.NIC_TYPE_IPMI {
+				continue
+			}
+			rIdx, rNic := findRemoteNic(nic.GetMac())
+			if rNic == nil {
+				return nil, errors.Errorf("Not found remote nic by mac %q", nic.GetMac())
+			}
+			if nics[idx].Index != int32(rIdx) {
+				log.Infof("Reindex nic %s index %d to %d", nics[idx].GetMac(), nics[idx].Index, rIdx)
+				nics[idx].Index = int32(rIdx)
+			}
+		}
+		return nics, nil
+	}
+	nics, err := reIndexNics(desc.Nics)
+	if err != nil {
+		return nil, errors.Wrap(err, "Reindex nics")
+	}
+	desc.Nics = nics
+	sNics, err := reIndexNics(desc.NicsStandby)
+	if err != nil {
+		return nil, errors.Wrap(err, "Reindex standby nics")
+	}
+	desc.NicsStandby = sNics
+	return desc, nil
 }
 
 func (s *SBaremetalServer) GetNics() []types.SServerNic {

pkg/baremetal/options/options.go

@@ -36,6 +36,7 @@ type BaremetalOptions struct {
 	TftpMaxTimeoutRetries  int    `default:"50" help:"Maximal tftp timeout retries, default is 50"`
 	LengthyWorkerCount     int    `default:"8" help:"Parallel worker count for lengthy tasks"`
 	ShortWorkerCount       int    `default:"8" help:"Parallel worker count for short-lived tasks"`
+	TaskWorkerCount        int    `default:"32" help:"Parallel worker count for tasks"`
 
 	DefaultIpmiPassword       string `help:"Default IPMI passowrd"`
 	DefaultStrongIpmiPassword string `help:"Default strong IPMI passowrd"`

pkg/baremetal/pxe/dhcp.go

@@ -79,6 +79,7 @@ func (h *DHCPHandler) ServeDHCP(pkt dhcp.Packet, _ *net.UDPAddr, _ *net.Interfac
 	if req.RelayAddr.String() == "0.0.0.0" {
 		return nil, nil, fmt.Errorf("Request not from a DHCP relay, ignore mac: %s", req.ClientMac)
 	}
+	log.Infof("[DHCP] from relay %s packet, mac: %s", req.RelayAddr, req.ClientMac)
 	conf, targets, err := req.fetchConfig(h.baremetalManager.GetClientSession())
 	if err != nil {
 		return nil, nil, err
@@ -188,7 +189,7 @@ func (req *dhcpRequest) fetchConfig(session *mcclient.ClientSession) (*dhcp.Resp
 	if len(netConf.GuestDhcp) > 0 {
 		dhcpAddrList = strings.Split(netConf.GuestDhcp, ",")
 	}
-	log.Debugf("dhcpAddrList %s", dhcpAddrList)
+	log.Debugf("find network config %#v, dhcpAddrList %s", netConf, dhcpAddrList)
 
 	// TODO: set cache for netConf
 	if req.isPXERequest() {
@@ -227,6 +228,7 @@ func (req *dhcpRequest) fetchConfig(session *mcclient.ClientSession) (*dhcp.Resp
 			// from guestdhcp import GuestDHCPHelperTask
 			// task = GuestDHCPHelperTask(self)
 			// task.start()
+			log.Infof("Not found baremetal by mac: %s", req.ClientMac)
 			return nil, nil, nil
 		}
 		req.baremetalInstance = bmInstance

pkg/baremetal/tasks/baseprepare.go

@@ -670,6 +670,10 @@ func getDMIMemInfo(cli *ssh.Client) (*types.SDMIMemInfo, error) {
 	return sysutils.ParseDMIMemInfo(ret), nil
 }
 
+func GetNicsInfo(cli *ssh.Client) ([]*types.SNicDevInfo, error) {
+	return getNicsInfo(cli)
+}
+
 func getNicsInfo(cli *ssh.Client) ([]*types.SNicDevInfo, error) {
 	ret, err := cli.Run("/lib/mos/lsnic")
 	if err != nil {