Merge pull request #14970 from swordqiu/automated-cherry-pick-of-#14967-upstream-release-3.7

Automated cherry pick of #14967: fix: set vfs_cache_pressure to 350

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,4 +18,6 @@ assignees: ''
 **Environment**:
 - OS (e.g: `cat /etc/os-release`):
 - Kernel (e.g. `uname -a`):
-- Others:
+<!--
+- Version (e.g. `climc version-list`):
+-->

Makefile

@@ -268,7 +268,8 @@ dep:
 	@$(MAKE) mod
 
 mod:
-	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
+	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v sqlchemy))
+	go get -d yunion.io/x/sqlchemy@v1.0.0
 	go mod tidy
 	go mod vendor -v
 
@@ -330,7 +331,7 @@ image:
 .PHONY: image
 
 image-telegraf-raid-plugin:
-	VERSION=release-1.6.1 ARCH=all make image telegraf-raid-plugin
+	VERSION=release-1.6.3 ARCH=all make image telegraf-raid-plugin
 
 %:
 	@:

Makefile.common.mk

@@ -7,8 +7,8 @@ endif
 ModBaseName:=$(notdir $(ModName))
 
 DockerImageRegistry?=registry.cn-beijing.aliyuncs.com
-DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.0-5
-DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-4
+DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.1-5
+DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-5
 
 
 EnvIf=$(if $($(1)),$(1)=$($(1)))

OWNERS

@@ -4,12 +4,6 @@ reviewers:
   - ioito
   - swordqiu
   - tb365
-  - wanyaoqi
-  - yousong
-  - zhaoxiangchun
-  - zhasm
 approvers:
   - swordqiu
-  - yousong
-  - wanyaoqi
   - Zexi

build/docker/Dockerfile.ansibleserver

@@ -1,7 +1,7 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.3
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.5
 
 # install playbook and telegraf install pkg
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.2 /opt/yunion/playbook /opt/yunion/playbook
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.1.2 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/playbook /opt/yunion/playbook
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
 
 ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/apigateway /opt/yunion/bin/apigateway
+ADD ./_output/alpine-build/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.baremetal-agent

@@ -1,9 +1,11 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.3
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
 RUN mkdir -p /opt/yunion/bin
 
-ADD ./_output/bin/baremetal-agent /opt/yunion/bin/baremetal-agent
-ADD ./_output/bin/.baremetal-agent.bin /opt/yunion/bin/.baremetal-agent.bin
-ADD ./_output/bin/bundles/baremetal-agent /opt/yunion/bin/bundles/baremetal-agent
+#ADD ./_output/bin/baremetal-agent /opt/yunion/bin/baremetal-agent
+#ADD ./_output/bin/.baremetal-agent.bin /opt/yunion/bin/.baremetal-agent.bin
+#ADD ./_output/bin/bundles/baremetal-agent /opt/yunion/bin/bundles/baremetal-agent
+RUN apk add librados librbd
+ADD ./_output/alpine-build/bin/baremetal-agent /opt/yunion/bin/baremetal-agent

build/docker/Dockerfile.baremetal-base

@@ -1,7 +1,8 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
-RUN yum install -y https://iso.yunion.cn/3.6/rpms/packages/host/baremetal-pxerom-1.1.0-20072810.x86_64.rpm
+FROM --platform=linux/amd64 registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
+RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
+#RUN yum install -y http://192.168.23.50:8083/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 
-FROM frolvlad/alpine-glibc:glibc-2.28
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
 

build/docker/Dockerfile.climc

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:20210508
+FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:20210901
 
 ADD ./build/climc/root/opt /opt
 

build/docker/Dockerfile.climc-base

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
@@ -8,7 +8,7 @@ RUN mkdir -p /opt/yunion/bin
 
 RUN echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >>/etc/apk/repositories
 
-RUN apk add --no-cache bash bash-completion tzdata ca-certificates vim ipmitool kubectl && \
+RUN apk add --no-cache bash bash-completion tzdata ca-certificates vim ipmitool kubectl ceph-common && \
     rm -rf /var/cache/apk/*
 
 RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/cloudevent /opt/yunion/bin/cloudevent
+ADD ./_output/alpine-build/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudid

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/cloudid/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/cloudid /opt/yunion/bin/cloudid

build/docker/Dockerfile.cloudmon

@@ -0,0 +1,8 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+RUN mkdir -p /etc/yunion/data/
+
+ADD   ./_output/alpine-build/bin/cloudmon \
+      /opt/yunion/bin/

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/cloudnet /opt/yunion/bin/cloudnet
+ADD ./_output/alpine-build/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.cloudproxy

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/cloudproxy /opt/yunion/bin/cloudproxy

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/devtool /opt/yunion/bin/devtool
+ADD ./_output/alpine-build/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 

build/docker/Dockerfile.file-repo

@@ -11,11 +11,10 @@ RUN set -x \
 
 # install default playbook and install pkg
 Run mkdir -p /opt/yunion/ansible-install-pkg
-Run wget https://dl.influxdata.com/telegraf/releases/telegraf-1.17.0-1.x86_64.rpm -P /opt/yunion/ansible-install-pkg
-Run wget https://dl.influxdata.com/telegraf/releases/telegraf-1.17.0-1.aarch64.rpm -P /opt/yunion/ansible-install-pkg
-Run wget https://dl.influxdata.com/telegraf/releases/telegraf-1.17.0_windows_amd64.zip -P /opt/yunion/ansible-install-pkg
-Run wget https://dl.influxdata.com/telegraf/releases/telegraf_1.17.0-1_amd64.deb -P /opt/yunion/ansible-install-pkg
-Run wget https://dl.influxdata.com/telegraf/releases/telegraf_1.17.0-1_arm64.deb -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.19.2-yn~fe11a96b-0.aarch64.rpm -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf-1.19.2-yn~fe11a96b-0.x86_64.rpm -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.19.2-yn~fe11a96b-0_amd64.deb -P /opt/yunion/ansible-install-pkg
+Run wget https://yunioniso.oss-cn-beijing.aliyuncs.com/rpms/telegraf/telegraf_1.19.2-yn~fe11a96b-0_arm64.deb -P /opt/yunion/ansible-install-pkg
 
 Run mkdir -p /opt/yunion/playbook
 Run mkdir /opt/yunion/playbook/monitor-agent

build/docker/Dockerfile.glance

@@ -1,7 +1,7 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/torrent:20200820
+FROM registry.cn-beijing.aliyuncs.com/yunionio/torrent:20210815.0
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
 RUN apk add --no-cache s3fs-fuse --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
 RUN mkdir -p /opt/yunion/bin
-ADD ./_output/alpine-build/bin/glance /opt/yunion/bin/glance
+ADD ./_output/alpine-build/bin/glance /opt/yunion/bin/glance

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/keystone /opt/yunion/bin/keystone
+ADD ./_output/alpine-build/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/logger /opt/yunion/bin/logger

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/monitor/root/opt/ /opt/
-ADD ./_output/alpine-build/bin/monitor /opt/yunion/bin/monitor
+ADD ./_output/alpine-build/bin/monitor /opt/yunion/bin/monitor

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -1,4 +1,5 @@
-FROM alpine:3.11
+# TODO: use 3.14 will lead 'apk add qemu-img fail on arm64'
+FROM alpine:3.13.5
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.region-dns

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/region-dns /opt/yunion/bin/region-dns

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/s3gateway /opt/yunion/bin/s3gateway
+ADD ./_output/alpine-build/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-ADD ./_output/alpine-build/bin/scheduler /opt/yunion/bin/scheduler
+ADD ./_output/alpine-build/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.telegraf-raid-plugin

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.1
 
 ADD ./_output/centos-build/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/Dockerfile.torrent

@@ -1,15 +1,15 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
+MAINTAINER "Zexi Li <zexi.li@icloud.com>"
+
+# HACK: use v3.14 alpine mirror to upgrade qemu-img
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/main > /etc/apk/repositories
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/community >> /etc/apk/repositories
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates qemu-img && \
+    apk add --no-cache qemu-img && \
     rm -rf /var/cache/apk/*
 
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV TZ Asia/Shanghai
-
 # add executable file torrent
 # make cmd/torrent
 RUN mkdir -p /opt/yunion/bin

build/docker/Dockerfile.yunionconf

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 ADD ./_output/alpine-build/bin/yunionconf /opt/yunion/bin/yunionconf

build/docker/Makefile

@@ -1,7 +1,35 @@
+REGISTRY ?= "registry.cn-beijing.aliyuncs.com/yunionio"
+DOCKER_BUILD = docker build -t $(REGISTRY)
+DOCKER_BUILDX = docker buildx build --platform linux/arm64,linux/amd64 --push -t $(REGISTRY)
+
+
 debian10-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/debian10-base:1.0 -f ./Dockerfile.debian-base .
 
+ONECLOUD_BASE_VERSION = v0.3.5
+
+onecloud-base:
+	$(DOCKER_BUILDX)/onecloud-base:$(ONECLOUD_BASE_VERSION) -f ./Dockerfile.onecloud-base .
+
+ANSIBLESERVER_BASE = v1.0.5
+
+ansibleserver-base:
+	$(DOCKER_BUILDX)/ansibleserver-base:$(ANSIBLESERVER_BASE) -f ./Dockerfile.ansibleserver-base .
+
+CLIMC_BASE_VERSION = 20210901
+
 climc-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
-		-t registry.cn-beijing.aliyuncs.com/yunionio/climc-base:$(VERSION) -f ./Dockerfile.climc-base .
+		-t registry.cn-beijing.aliyuncs.com/yunionio/climc-base:$(CLIMC_BASE_VERSION) -f ./Dockerfile.climc-base .
+
+BAREMETAL_BASE_VERSION = v0.3.3
+
+baremetal-base:
+	$(DOCKER_BUILDX)/baremetal-base:$(BAREMETAL_BASE_VERSION) -f ./Dockerfile.baremetal-base .
+	#docker push $(REGISTRY)/baremetal-base:$(BAREMETAL_BASE_VERSION)
+
+FILE_REPO_VERSION = v0.3.3
+
+file-repo:
+	$(DOCKER_BUILDX)/file-repo:$(FILE_REPO_VERSION) -f ./Dockerfile.file-repo .

build/docker/multi-arch/Dockerfile.torrent

@@ -1,24 +1,24 @@
 FROM registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5 as build
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
+
 RUN mkdir -p /root/go/src/yunion.io/x/onecloud
 COPY . /root/go/src/yunion.io/x/onecloud
+RUN cd /root/go/src/yunion.io/x/onecloud && make cmd/torrent
 
-WORKDIR /root/go/src/yunion.io/x/onecloud
-RUN make cmd/torrent
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+MAINTAINER "Zexi Li <zexi.li@icloud.com>"
 
-MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
+# HACK: use v3.14 alpine mirror to upgrade qemu-img
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/main > /etc/apk/repositories
+RUN echo http://dl-cdn.alpinelinux.org/alpine/v3.14/community >> /etc/apk/repositories
 
 RUN apk update && \
-    apk add --no-cache tzdata ca-certificates qemu-img && \
+    apk add --no-cache qemu-img && \
     rm -rf /var/cache/apk/*
 
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV TZ Asia/Shanghai
-
+# TAG=20210815.0
 # add executable file torrent
 # make cmd/torrent
 RUN mkdir -p /opt/yunion/bin

build/docker/multi-arch/Dockerfile.torrent.dockerignore

@@ -0,0 +1,2 @@
+_output/alpine-build/_cache
+

cmd/climc/shell/cloudevent/cloudevents.go

@@ -40,4 +40,18 @@ func init() {
 		printList(result, modules.Cloudevents.GetColumns(s))
 		return nil
 	})
+
+	type CloudeventLogsPurgeOptions struct {
+	}
+	R(&CloudeventLogsPurgeOptions{}, "cloud-event-purge", "Purge obsolete cloud event logs", func(s *mcclient.ClientSession, opts *CloudeventLogsPurgeOptions) error {
+		_, err := modules.Cloudevents.PerformClassAction(s, "purge-splitable", nil)
+		if err != nil {
+			return err
+		}
+		_, err = modules.CloudeventLogs.PerformClassAction(s, "purge-splitable", nil)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
 }

cmd/climc/shell/cloudproxy/proxy_agents.go

@@ -28,4 +28,5 @@ func init() {
 	cmd.List(&options.ProxyAgentListOptions{})
 	cmd.Show(&options.ProxyAgentShowOptions{})
 	cmd.Delete(&options.ProxyAgentDeleteOptions{})
+	cmd.Perform("purge-forwards", &options.ProxyAgentPurgeForwardsOptions{})
 }

cmd/climc/shell/compute/cloudaccounts.go

@@ -37,6 +37,7 @@ func init() {
 	cmd.CreateWithKeyword("create-aws", &options.SAWSCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-openstack", &options.SOpenStackCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-huawei", &options.SHuaweiCloudAccountCreateOptions{})
+	cmd.CreateWithKeyword("create-hcso", &options.SHCSOAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-ucloud", &options.SUcloudCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-zstack", &options.SZStackCloudAccountCreateOptions{})
 	cmd.CreateWithKeyword("create-s3", &options.SS3CloudAccountCreateOptions{})
@@ -54,6 +55,7 @@ func init() {
 	cmd.UpdateWithKeyword("update-aws", &options.SAWSCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-openstack", &options.SOpenStackCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-huawei", &options.SHuaweiCloudAccountUpdateOptions{})
+	cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{})
 	cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{})
@@ -69,6 +71,7 @@ func init() {
 	cmd.PerformWithKeyword("update-credential-aws", "update-credential", &options.SAWSCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-openstack", "update-credential", &options.SOpenStackCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-huawei", "update-credential", &options.SHuaweiCloudAccountUpdateCredentialOptions{})
+	cmd.PerformWithKeyword("update-credential-hcso", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-ucloud", "update-credential", &options.SUcloudCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-zstack", "update-credential", &options.SZStackCloudAccountUpdateCredentialOptions{})
 	cmd.PerformWithKeyword("update-credential-s3", "update-credential", &options.SS3CloudAccountUpdateCredentialOptions{})

cmd/climc/shell/compute/cloudproviders.go

@@ -18,251 +18,38 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
+	cmd := shell.NewResourceCmd(&modules.Cloudproviders).WithKeyword("cloud-provider")
+	cmd.List(&compute.CloudproviderListOptions{})
+	cmd.Update(&compute.CloudproviderUpdateOptions{})
+	cmd.Show(&options.BaseIdOptions{})
+	cmd.Delete(&options.BaseIdOptions{})
+	cmd.Perform("change-project", &compute.CloudproviderChangeProjectOptions{})
+	cmd.Perform("enable", &options.BaseIdOptions{})
+	cmd.Perform("disable", &options.BaseIdOptions{})
+	cmd.Perform("sync", &compute.CloudproviderSyncOptions{})
+	cmd.Perform("project-mapping", &compute.ClouproviderProjectMappingOptions{})
 
-	type CloudproviderListOptions struct {
-		options.BaseListOptions
-
-		Usable bool `help:"Vpc & Network usable"`
-
-		HasObjectStorage bool     `help:"filter cloudproviders that has object storage"`
-		NoObjectStorage  bool     `help:"filter cloudproviders that has no object storage"`
-		Capability       []string `help:"capability filter" choices:"project|compute|network|loadbalancer|objectstore|rds|cache|event"`
-		Cloudregion      string   `help:"filter cloudproviders by cloudregion"`
-
-		HostSchedtagId string `help:"filter by host schedtag"`
-	}
-	R(&CloudproviderListOptions{}, "cloud-provider-list", "List cloud providers", func(s *mcclient.ClientSession, args *CloudproviderListOptions) error {
-		var params *jsonutils.JSONDict
-		{
-			var err error
-			params, err = args.BaseListOptions.Params()
-			if err != nil {
-				return err
-			}
-
-			if args.Usable {
-				params.Add(jsonutils.NewBool(true), "usable")
-			}
-
-			if args.HasObjectStorage {
-				params.Add(jsonutils.JSONTrue, "has_object_storage")
-			} else if args.NoObjectStorage {
-				params.Add(jsonutils.JSONFalse, "has_object_storage")
-			}
-
-			if len(args.Capability) > 0 {
-				params.Add(jsonutils.NewStringArray(args.Capability), "capability")
-			}
-
-			if len(args.Cloudregion) > 0 {
-				params.Add(jsonutils.NewString(args.Cloudregion), "cloudregion")
-			}
-
-			if len(args.HostSchedtagId) > 0 {
-				params.Add(jsonutils.NewString(args.HostSchedtagId), "host_schedtag_id")
-			}
-		}
-		result, err := modules.Cloudproviders.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.Cloudproviders.GetColumns(s))
-		return nil
-	})
-
-	type CloudproviderUpdateOptions struct {
-		ID        string `help:"ID or Name of cloud provider"`
-		Name      string `help:"New name to update"`
-		AccessUrl string `help:"New access url"`
-		Desc      string `help:"Description"`
-	}
-	R(&CloudproviderUpdateOptions{}, "cloud-provider-update", "Update a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderUpdateOptions) error {
-		params := jsonutils.NewDict()
-		if len(args.Name) > 0 {
-			params.Add(jsonutils.NewString(args.Name), "name")
-		}
-		if len(args.AccessUrl) > 0 {
-			params.Add(jsonutils.NewString(args.AccessUrl), "access_url")
-		}
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
-		}
-		if params.Size() == 0 {
-			return InvalidUpdateError()
-		}
-		result, err := modules.Cloudproviders.Update(s, args.ID, params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type CloudproviderChangeProjectOptions struct {
-		ID     string `help:"ID or Name of cloud provider"`
-		TENANT string `help:"ID or Name of tenant"`
-	}
-	R(&CloudproviderChangeProjectOptions{}, "cloud-provider-change-project", "Change project for provider", func(s *mcclient.ClientSession, args *CloudproviderChangeProjectOptions) error {
-		result, err := modules.Cloudproviders.PerformAction(s, args.ID, "change-project", jsonutils.Marshal(map[string]string{"project": args.TENANT}))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type CloudproviderShowOptions struct {
-		ID string `help:"ID or Name of cloud provider"`
-	}
-	R(&CloudproviderShowOptions{}, "cloud-provider-show", "Get details of a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.Get(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&CloudproviderShowOptions{}, "cloud-provider-delete", "Delete a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.Delete(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&CloudproviderShowOptions{}, "cloud-provider-enable", "Enable cloud provider", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.PerformAction(s, args.ID, "enable", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&CloudproviderShowOptions{}, "cloud-provider-disable", "Disable cloud provider", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.PerformAction(s, args.ID, "disable", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&CloudproviderShowOptions{}, "cloud-provider-balance", "Get balance", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.GetSpecific(s, args.ID, "balance", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type CloudproviderUpdateCredentialOptions struct {
-		ID      string `help:"ID or Name of cloud provider"`
-		ACCOUNT string `help:"new account"`
-		SECRET  string `help:"new secret"`
-	}
-	R(&CloudproviderUpdateCredentialOptions{}, "cloud-provider-update-credential", "Update credential of a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderUpdateCredentialOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.ACCOUNT), "account")
-		params.Add(jsonutils.NewString(args.SECRET), "secret")
-
-		result, err := modules.Cloudproviders.PerformAction(s, args.ID, "update-credential", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type CloudproviderSyncOptions struct {
-		ID          string   `help:"ID or Name of cloud provider"`
-		Force       bool     `help:"Force sync no matter what"`
-		FullSync    bool     `help:"Synchronize everything"`
-		ProjectSync bool     `help:"Auto sync project info"`
-		Region      []string `help:"region to sync"`
-		Zone        []string `help:"region to sync"`
-		Host        []string `help:"region to sync"`
-	}
-	R(&CloudproviderSyncOptions{}, "cloud-provider-sync", "Sync of a cloud provider account", func(s *mcclient.ClientSession, args *CloudproviderSyncOptions) error {
-		params := jsonutils.NewDict()
-		if args.Force {
-			params.Add(jsonutils.JSONTrue, "force")
-		}
-		if args.FullSync {
-			params.Add(jsonutils.JSONTrue, "full_sync")
-		}
-		if args.ProjectSync {
-			params.Add(jsonutils.JSONTrue, "project_sync")
-		}
-		if len(args.Region) > 0 {
-			params.Add(jsonutils.NewStringArray(args.Region), "region")
-		}
-		if len(args.Zone) > 0 {
-			params.Add(jsonutils.NewStringArray(args.Zone), "zone")
-		}
-		if len(args.Host) > 0 {
-			params.Add(jsonutils.NewStringArray(args.Host), "host")
-		}
-		result, err := modules.Cloudproviders.PerformAction(s, args.ID, "sync", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type CloudproviderClientRCOptions struct {
-		ID string `help:"ID or name of cloud provider"`
-	}
-	R(&CloudproviderClientRCOptions{}, "cloud-provider-clirc", "Get client RC file of the cloud provider", func(s *mcclient.ClientSession, args *CloudproviderClientRCOptions) error {
-		result, err := modules.Cloudproviders.GetSpecific(s, args.ID, "clirc", nil)
-		if err != nil {
-			return err
-		}
+	cmd.GetWithCustomShow("clirc", func(result jsonutils.JSONObject) {
 		rc := make(map[string]string)
-		err = result.Unmarshal(&rc)
+		err := result.Unmarshal(&rc)
 		if err != nil {
-			return err
+			log.Errorf("error: %v", err)
+			return
 		}
 		for k, v := range rc {
 			fmt.Printf(`export %s="%s"`+"\n", k, v)
 		}
-		return nil
-	})
-
-	type CloudproviderStorageClassesOptions struct {
-		ID          string `help:"ID or Name of cloud provider" json:"-"`
-		Cloudregion string `help:"cloud region name or Id"`
-	}
-	R(&CloudproviderStorageClassesOptions{}, "cloud-provider-storage-classes", "Get list of supported storage classes of a cloud provider", func(s *mcclient.ClientSession, args *CloudproviderStorageClassesOptions) error {
-		params, err := options.StructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.Cloudproviders.GetSpecific(s, args.ID, "storage-classes", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&CloudproviderShowOptions{}, "cloud-provider-change-owner-candidate-domains", "Show candiate domains of a cloud provider  changing owner", func(s *mcclient.ClientSession, args *CloudproviderShowOptions) error {
-		result, err := modules.Cloudproviders.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
+	}, &options.BaseIdOptions{})
+	cmd.Get("storage-classes", &compute.CloudproviderStorageClassesOptions{})
+	cmd.Get("change-owner-candidate-domains", &options.BaseIdOptions{})
+	cmd.Get("balance", &options.BaseIdOptions{})
 }

cmd/climc/shell/compute/hosts.go

@@ -15,164 +15,51 @@
 package compute
 
 import (
+	"context"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
 func init() {
-	type HostListOptions struct {
-		Schedtag  string `help:"List hosts in schedtag"`
-		Zone      string `help:"List hosts in zone"`
-		Region    string `help:"List hosts in region"`
-		Wire      string `help:"List hosts in wire"`
-		Image     string `help:"List hosts cached images" json:"cachedimage"`
-		Storage   string `help:"List hosts attached to storages"`
-		Baremetal string `help:"List hosts that is managed by baremetal system" choices:"true|false"`
-		Empty     bool   `help:"show empty host" json:"-"`
-		Occupied  bool   `help:"show occupid host" json:"-"`
-		Enabled   bool   `help:"Show enabled host only" json:"-"`
-		Disabled  bool   `help:"Show disabled host only" json:"-"`
-		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud|google|ctyun"`
-		AnyMac    string `help:"Mac matches one of the host's interface"`
-		AnyIp     string `help:"IP matches one of the host's interface"`
+	cmd := shell.NewResourceCmd(&modules.Hosts)
+	cmd.List(&compute.HostListOptions{})
+	cmd.Show(&options.BaseShowOptions{})
+	cmd.GetMetadata(&options.BaseIdOptions{})
+	cmd.GetProperty(&compute.HostStatusStatisticsOptions{})
 
-		IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"`
+	cmd.Perform("ping", &options.BaseIdOptions{})
+	cmd.Perform("undo-convert", &options.BaseIdOptions{})
+	cmd.Perform("maintenance", &options.BaseIdOptions{})
+	cmd.Perform("unmaintenance", &options.BaseIdOptions{})
+	cmd.Perform("start", &options.BaseIdOptions{})
+	cmd.Perform("stop", &options.BaseIdOptions{})
+	cmd.Perform("reset", &options.BaseIdOptions{})
+	cmd.BatchDelete(&options.BaseIdsOptions{})
+	cmd.Perform("remove-all-netifs", &options.BaseIdOptions{})
 
-		ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"`
+	cmd.BatchPerform("enable", &options.BaseIdsOptions{})
+	cmd.BatchPerform("disable", &options.BaseIdsOptions{})
+	cmd.BatchPerform("syncstatus", &options.BaseIdsOptions{})
+	cmd.BatchPerform("sync-config", &options.BaseIdsOptions{})
+	cmd.BatchPerform("prepare", &options.BaseIdsOptions{})
+	cmd.BatchPerform("ipmi-probe", &options.BaseIdsOptions{})
 
-		Usable *bool `help:"List all zones that is usable"`
+	cmd.Get("ipmi", &options.BaseIdOptions{})
+	cmd.Get("vnc", &options.BaseIdOptions{})
 
-		Hypervisor string `help:"filter hosts by hypervisor"`
-
-		StorageNotAttached bool `help:"List hosts not attach specified storage"`
-
-		Uuid string `help:"find host with given system uuid"`
-
-		CdromBoot *bool `help:"filter hosts list by cdrom_boot=true|false"`
-
-		Sn string `help:"find host by sn"`
-
-		OrderByServerCount string `help:"Order by server count" choices:"desc|asc"`
-
-		options.BaseListOptions
-	}
-	R(&HostListOptions{}, "host-list", "List hosts", func(s *mcclient.ClientSession, opts *HostListOptions) error {
-		params, err := options.ListStructToParams(opts)
-		if err != nil {
-			return err
-		}
-
-		if opts.Empty {
-			params.Add(jsonutils.JSONTrue, "is_empty")
-		} else if opts.Occupied {
-			params.Add(jsonutils.JSONFalse, "is_empty")
-		}
-		if opts.Enabled {
-			params.Add(jsonutils.NewInt(1), "enabled")
-		} else if opts.Disabled {
-			params.Add(jsonutils.NewInt(0), "enabled")
-		}
-		if len(opts.Uuid) > 0 {
-			params.Add(jsonutils.NewString(opts.Uuid), "uuid")
-		}
-		if len(opts.Sn) > 0 {
-			params.Add(jsonutils.NewString(opts.Sn), "sn")
-		}
-		result, err := modules.Hosts.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	type HostDetailOptions struct {
-		ID string `help:"ID or name of host"`
-	}
-	R(&HostDetailOptions{}, "host-show", "Show details of a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.Get(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-ping", "Ping a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "ping", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-metadata", "Show metadata of a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.GetMetadata(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type HostOpsOptions struct {
-		ID []string `help:"ID or name of hosts"`
-	}
-	R(&HostOpsOptions{}, "host-enable", "Enable a host", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "enable", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-disable", "Disable a host", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "disable", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-syncstatus", "Synchronize status of a host", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "syncstatus", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-sync-config", "Synchronize config of a host", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "sync-config", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-prepare", "Prepare a host for installation", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "prepare", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-ipmi-probe", "Do ipmi probe host", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchPerformAction(s, args.ID, "ipmi-probe", nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-ipmi", "Get IPMI information of a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.GetSpecific(s, args.ID, "ipmi", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-logininfo", "Get SSH login information of a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
+	R(&options.BaseIdOptions{}, "host-logininfo", "Get SSH login information of a host", func(s *mcclient.ClientSession, args *options.BaseIdOptions) error {
 		srvid, e := modules.Hosts.GetId(s, args.ID, nil)
 		if e != nil {
 			return e
@@ -185,15 +72,6 @@ func init() {
 		return nil
 	})
 
-	R(&HostDetailOptions{}, "host-vnc", "Get VNC information of a host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.GetSpecific(s, args.ID, "vnc", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
 	type HostPropertyOptions struct {
 	}
 
@@ -206,7 +84,7 @@ func init() {
 		return nil
 	})
 
-	R(&HostListOptions{}, "host-node-count", "Get host node count", func(s *mcclient.ClientSession, opts *HostListOptions) error {
+	R(&compute.HostListOptions{}, "host-node-count", "Get host node count", func(s *mcclient.ClientSession, opts *compute.HostListOptions) error {
 		params, err := options.ListStructToParams(opts)
 		if err != nil {
 			return err
@@ -348,67 +226,6 @@ func init() {
 		printObject(result)
 		return nil
 	})
-
-	R(&HostDetailOptions{}, "host-undo-convert", "Undo converting a host to hypervisor", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "undo-convert", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-maintenance", "Reboot host into PXE offline OS, do maintenance jobs", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "maintenance", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-unmaintenance", "Reboot host back into disk installed OS", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "unmaintenance", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-start", "Power on host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "start", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-stop", "Power off host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "stop", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostDetailOptions{}, "host-reset", "Power reset host", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "reset", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&HostOpsOptions{}, "host-delete", "Delete host record", func(s *mcclient.ClientSession, args *HostOpsOptions) error {
-		results := modules.Hosts.BatchDelete(s, args.ID, nil)
-		printBatchResults(results, modules.Hosts.GetColumns(s))
-		return nil
-	})
-
 	type HostAddNetIfOptions struct {
 		ID        string `help:"ID or Name of host"`
 		WIRE      string `help:"ID or Name of wire to attach"`
@@ -506,15 +323,6 @@ func init() {
 		return nil
 	})
 
-	R(&HostDetailOptions{}, "host-remove-all-netifs", "Remvoe all netifs expect admin&ipmi netifs", func(s *mcclient.ClientSession, args *HostDetailOptions) error {
-		result, err := modules.Hosts.PerformAction(s, args.ID, "remove-all-netifs", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
 	type HostCacheImageActionOptions struct {
 		ID     string `help:"ID or name of host"`
 		IMAGE  string `help:"ID or name of image"`
@@ -718,9 +526,36 @@ func init() {
 			return e
 		}
 		i, e := modules.Hosts.GetLoginInfo(s, srvid, nil)
+		privateKey := ""
 		if e != nil {
-			return e
+			if httputils.ErrorCode(e) == 404 {
+				var err error
+				privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+				if err != nil {
+					return errors.Wrap(err, "fetch private key")
+				}
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewString(string(args.ID)), "ID")
+				ret, err := modules.Hosts.Get(s, args.ID, params)
+				if err != nil {
+					return errors.Wrap(err, "get host by ID")
+				}
+				ip, err := ret.GetString("access_ip")
+				if err != nil {
+					return errors.Wrap(err, "get the ip of the host")
+				}
+
+				jsonItem := jsonutils.NewDict()
+				jsonItem.Add(jsonutils.NewString(ip), "ip")
+				jsonItem.Add(jsonutils.NewString("root"), "username")
+				jsonItem.Add(jsonutils.NewString(""), "password")
+				jsonItem.Add(jsonutils.NewInt(22), "port")
+				i = jsonItem
+			} else {
+				return e
+			}
 		}
+
 		host, err := i.GetString("ip")
 		if err != nil {
 			return err
@@ -737,7 +572,7 @@ func init() {
 		if args.Port != 22 {
 			port = args.Port
 		}
-		sshCli, err := ssh.NewClient(host, port, user, passwd, "")
+		sshCli, err := ssh.NewClient(host, port, user, passwd, privateKey)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/compute/server_forward.go

@@ -15,7 +15,11 @@
 package compute
 
 import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 )
@@ -26,3 +30,63 @@ func init() {
 	cmd.Perform("close-forward", &options.ServerCloseForwardOptions{})
 	cmd.Perform("list-forward", &options.ServerListForwardOptions{})
 }
+
+type forwardInfo struct {
+	ProxyAddr string `json:"proxy_addr"`
+	ProxyPort int    `json:"proxy_port"`
+}
+
+func dump(input jsonutils.JSONObject) (*forwardInfo, error) {
+	ret := new(forwardInfo)
+	if err := input.Unmarshal(ret); err != nil {
+		return nil, errors.Wrap(err, "JSONObject unmarshal failed")
+	}
+
+	if ret.ProxyAddr == "" {
+		return nil, errors.Errorf("proxy_addr is empty")
+	}
+
+	if ret.ProxyPort <= 0 {
+		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyPort)
+	}
+
+	return ret, nil
+}
+
+func openForward(session *mcclient.ClientSession, srvid string) (*forwardInfo, error) {
+	opt := &options.ServerOpenForwardOptions{
+		ServerIdOptions: options.ServerIdOptions{
+			ID: srvid,
+		},
+		Proto: "tcp",
+		Port:  22,
+	}
+
+	params, err := opt.Params()
+	if err != nil {
+		return nil, errors.Wrap(err, "get open forward params")
+	}
+	jsonItem, err := modules.Servers.PerformAction(session, opt.ID, "open-forward", params)
+	if err != nil {
+		return nil, err
+	}
+
+	return dump(jsonItem)
+}
+
+func closeForward(session *mcclient.ClientSession, srvid string, fItem *forwardInfo) (jsonutils.JSONObject, error) {
+	opt := &options.ServerCloseForwardOptions{
+		ServerIdOptions: options.ServerIdOptions{
+			ID: srvid,
+		},
+		Proto:     "tcp",
+		ProxyAddr: fItem.ProxyAddr,
+		ProxyPort: fItem.ProxyPort,
+	}
+
+	params, err := opt.Params()
+	if err != nil {
+		return nil, errors.Wrap(err, "close forward params")
+	}
+	return modules.Servers.PerformAction(session, opt.ID, "close-forward", params)
+}

cmd/climc/shell/compute/servers.go

@@ -15,6 +15,7 @@
 package compute
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -87,6 +88,9 @@ func init() {
 	cmd.Perform("remote-update", new(options.ServerRemoteUpdateOptions))
 	cmd.Perform("create-eip", &options.ServerCreateEipOptions{})
 	cmd.Perform("make-sshable", &options.ServerMakeSshableOptions{})
+	cmd.Perform("migrate-network", &options.ServerMigrateNetworkOptions{})
+	cmd.Perform("set-sshport", &options.ServerSetSshportOptions{})
+	cmd.Perform("have-agent", &options.ServerHaveAgentOptions{})
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))
@@ -96,6 +100,10 @@ func init() {
 	cmd.Get("sshable", new(options.ServerIdOptions))
 	cmd.Get("make-sshable-cmd", new(options.ServerIdOptions))
 	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
+	cmd.Get("change-owner-candidate-domains", new(options.ServerChangeOwnerCandidateDomainsOptions))
+	cmd.Get("sshport", new(options.ServerIdOptions))
+
+	cmd.GetProperty(&options.ServerStatusStatisticsOptions{})
 
 	type ServerTaskShowOptions struct {
 		ID       string `help:"ID or name of server" json:"-"`
@@ -179,6 +187,19 @@ func init() {
 		return nil
 	})
 
+	R(&options.ResourceMetadataOptions{}, "server-set-metadata", "Set raw metadata of a server", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Servers.PerformAction(s, opts.ID, "metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.ServerCreateFromInstanceSnapshot{}, "server-create-from-instance-snapshot", "server create from instance snapshot",
 		func(s *mcclient.ClientSession, opts *options.ServerCreateFromInstanceSnapshot) error {
 			params := &compute.ServerCreateInput{}
@@ -777,6 +798,16 @@ func init() {
 			return err
 		}
 
+		eip, err := srv.GetString("eip")
+		if err != nil && err.Error() != "Get: key not found" {
+			return err
+		}
+
+		vpcid, err := srv.GetString("vpc_id")
+		if err != nil {
+			return err
+		}
+
 		address := make([]string, 0)
 		nics, err := srv.GetArray("nics")
 		if err != nil {
@@ -791,13 +822,15 @@ func init() {
 			return fmt.Errorf("Not found ip address from server %s", opts.ID)
 		}
 
+		privateKey := ""
 		params := jsonutils.NewDict()
 		if len(opts.Key) > 0 {
-			privateKey, e := ioutil.ReadFile(opts.Key)
+			key, e := ioutil.ReadFile(opts.Key)
 			if e != nil {
 				return e
 			}
-			params.Add(jsonutils.NewString(string(privateKey)), "private_key")
+			params.Add(jsonutils.NewString(string(key)), "private_key")
+			privateKey = string(key)
 		}
 
 		i, e := modules.Servers.GetLoginInfo(s, srvid, params)
@@ -805,37 +838,88 @@ func init() {
 			return e
 		}
 		passwd, err := i.GetString("password")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.Password != "" {
 			passwd = opts.Password
 		}
 		user, err := i.GetString("username")
-		if err != nil {
+		if err != nil && !opts.UseCloudroot {
 			return err
 		}
 		if opts.User != "" {
 			user = opts.User
 		}
-
-		host := address[0]
-		if opts.Host != "" {
-			host = opts.Host
-		}
 		port := 22
 		if opts.Port != 22 {
 			port = opts.Port
 		}
 
-		sshCli, err := ssh.NewClient(host, port, user, passwd, "")
-		if err != nil {
-			return err
+		var forwardItem *forwardInfo = nil
+		host := address[0]
+		if opts.Host != "" {
+			host = opts.Host
+		} else {
+			if eip != "" {
+				host = eip
+			} else {
+				if vpcid != "default" {
+					forwardItem, err = openForward(s, srvid)
+					if err != nil {
+						return err
+					}
+					host = forwardItem.ProxyAddr
+					port = forwardItem.ProxyPort
+				}
+			}
 		}
+
+		if opts.UseCloudroot {
+			var err error
+			privateKey, err = modules.Sshkeypairs.FetchPrivateKeyBySession(context.Background(), s)
+			if err != nil {
+				return err
+			}
+			passwd = ""
+			user = "cloudroot"
+		}
+
+		var sshCli *ssh.Client
+		err = nil
+		for ; sshCli == nil; sshCli, err = ssh.NewClient(host, port, user, passwd, privateKey) {
+			if err == nil {
+				continue
+			}
+			if opts.Host != "" {
+				return err
+			}
+			if forwardItem != nil {
+				closeForward(s, srvid, forwardItem)
+				return err
+			} else {
+				if vpcid != "default" {
+					forwardItem, e = openForward(s, srvid)
+					if e != nil {
+						return e
+					}
+					host = forwardItem.ProxyAddr
+					port = forwardItem.ProxyPort
+				}
+			}
+		}
+
 		log.Infof("ssh %s:%d", host, port)
 		if err := sshCli.RunTerminal(); err != nil {
+			if forwardItem != nil {
+				closeForward(s, srvid, forwardItem)
+			}
 			return err
 		}
+
+		if forwardItem != nil {
+			closeForward(s, srvid, forwardItem)
+		}
 		return nil
 	})
 

cmd/climc/shell/devtool/script.go

@@ -24,6 +24,7 @@ func init() {
 	cmd := shell.NewResourceCmd(&modules.DevToolScripts).WithKeyword("devtool-script")
 	cmd.List(new(options.ScriptListOptions))
 	cmd.Perform("apply", new(options.ScriptApplyOptions))
+	cmd.Perform("batch-apply", new(options.ScriptBatchApplyOptions))
 	cmd1 := shell.NewResourceCmd(&modules.DevToolScriptApplyRecords).WithKeyword("devtool-script-record")
 	cmd1.List(new(options.ScriptApplyRecordListOptions))
 }

cmd/climc/shell/devtool/serviceurl.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/devtool"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.DevToolServiceUrls).WithKeyword("devtool-serviceurl")
+	cmd.List(new(options.ServiceUrlListOptions))
+	cmd.Create(new(options.ServiceUrlCreateOptions))
+	cmd.Show(new(options.ServiceUrlOptions))
+	cmd.Delete(new(options.ServiceUrlOptions))
+}

cmd/climc/shell/devtool/sshinfo.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package devtool
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/devtool"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.DevToolSshInfos).WithKeyword("devtool-sshinfo")
+	cmd.List(new(options.SshInfoListOptions))
+	cmd.Create(new(options.SshInfoCreateOptions))
+	cmd.Show(new(options.SshInfoOptions))
+	cmd.Delete(new(options.SshInfoOptions))
+}

cmd/climc/shell/events/splitable.go

@@ -24,7 +24,7 @@ import (
 
 func init() {
 	type EventSplitableOptions struct {
-		Service string `help:"service" choices:"compute|identity|image" default:"compute"`
+		Service string `help:"service" choices:"compute|identity|image|log|cloudevent" default:"compute"`
 	}
 	R(&EventSplitableOptions{}, "logs-splitable", "Show splitable info of event table", func(s *mcclient.ClientSession, args *EventSplitableOptions) error {
 		var results jsonutils.JSONObject
@@ -34,6 +34,10 @@ func init() {
 			results, err = modules.IdentityLogs.Get(s, "splitable", nil)
 		case "image":
 			results, err = modules.ImageLogs.Get(s, "splitable", nil)
+		case "log":
+			results, err = modules.Actions.Get(s, "splitable", nil)
+		case "cloudevent":
+			results, err = modules.Cloudevents.Get(s, "splitable", nil)
 		default:
 			results, err = modules.Logs.Get(s, "splitable", nil)
 		}
@@ -58,6 +62,10 @@ func init() {
 			results, err = modules.IdentityLogs.PerformClassAction(s, "purge-splitable", nil)
 		case "image":
 			results, err = modules.ImageLogs.PerformClassAction(s, "purge-splitable", nil)
+		case "log":
+			results, err = modules.Actions.PerformClassAction(s, "purge-splitable", nil)
+		case "cloudevent":
+			results, err = modules.Cloudevents.PerformClassAction(s, "purge-splitable", nil)
 		default:
 			results, err = modules.Logs.PerformClassAction(s, "purge-splitable", nil)
 		}

cmd/climc/shell/helper.go

@@ -224,6 +224,28 @@ type IShowOpt interface {
 	IIdOpt
 }
 
+type IPropertyOpt interface {
+	IOpt
+	Property() string
+}
+
+func (cmd ResourceCmd) GetProperty(args IPropertyOpt) {
+	man := cmd.manager
+	callback := func(s *mcclient.ClientSession, args IPropertyOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := man.(modulebase.Manager).Get(s, args.Property(), params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	}
+	cmd.RunWithDesc(args.Property(), fmt.Sprintf("Get property of a %s", man.GetKeyword()), args, callback)
+}
+
 func (cmd ResourceCmd) Show(args IShowOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IShowOpt) error {
@@ -421,7 +443,7 @@ type IGetOpt interface {
 	IOpt
 }
 
-func (cmd ResourceCmd) Get(specific string, args IGetOpt) {
+func (cmd ResourceCmd) GetWithCustomShow(specific string, show func(data jsonutils.JSONObject), args IGetOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IGetOpt) error {
 		params, err := args.Params()
@@ -432,12 +454,16 @@ func (cmd ResourceCmd) Get(specific string, args IGetOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		show(ret)
 		return nil
 	}
 	cmd.RunWithDesc(specific, fmt.Sprintf("Get %s of a %s", specific, man.GetKeyword()), args, callback)
 }
 
+func (cmd ResourceCmd) Get(specific string, args IGetOpt) {
+	cmd.GetWithCustomShow(specific, printObject, args)
+}
+
 type IUpdateOpt interface {
 	IOpt
 	IIdOpt

cmd/climc/shell/identity/policies.go

@@ -24,13 +24,14 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/identity"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -74,25 +75,8 @@ func createPolicy(s *mcclient.ClientSession, name string, policy string, domain
 }
 
 func init() {
-	type PolicyListOptions struct {
-		options.BaseListOptions
-		Type          string `help:"filter by type"`
-		IsSystem      *bool  `help:"filter by is_system" negative:"is_no_system"`
-		Format        string `help:"policy format, default to yaml" default:"yaml" choices:"yaml|json"`
-		OrderByDomain string `help:"order by domain name" choices:"asc|desc"`
-	}
-	R(&PolicyListOptions{}, "policy-list", "List all policies", func(s *mcclient.ClientSession, args *PolicyListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.Policies.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.Policies.GetColumns(s))
-		return nil
-	})
+	cmd := shell.NewResourceCmd(&modules.Policies)
+	cmd.List(&identity.PolicyListOptions{})
 
 	type PolicyCreateOptions struct {
 		Domain   string `help:"domain of the policy"`

cmd/climc/shell/identity/roles.go

@@ -17,100 +17,21 @@ package identity
 import (
 	"yunion.io/x/jsonutils"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/identity"
 )
 
 func init() {
-	type RoleListOptions struct {
-		options.BaseListOptions
-		OrderByDomain string `help:"order by domain name" choices:"asc|desc"`
-	}
-	R(&RoleListOptions{}, "role-list", "List keystone Roles", func(s *mcclient.ClientSession, args *RoleListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.RolesV3.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.RolesV3.GetColumns(s))
-		return nil
-	})
-
-	type RoleDetailOptions struct {
-		ID     string `help:"ID or name of role"`
-		Domain string `help:"Domain"`
-	}
-	R(&RoleDetailOptions{}, "role-show", "Show details of a role", func(s *mcclient.ClientSession, args *RoleDetailOptions) error {
-		query := jsonutils.NewDict()
-		if len(args.Domain) > 0 {
-			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
-			if err != nil {
-				return err
-			}
-			query.Add(jsonutils.NewString(domainId), "domain_id")
-		}
-		role, err := modules.RolesV3.Get(s, args.ID, query)
-		if err != nil {
-			return err
-		}
-		printObject(role)
-		return nil
-	})
-	R(&RoleDetailOptions{}, "role-delete", "Delete a role", func(s *mcclient.ClientSession, args *RoleDetailOptions) error {
-		query := jsonutils.NewDict()
-		if len(args.Domain) > 0 {
-			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
-			if err != nil {
-				return err
-			}
-			query.Add(jsonutils.NewString(domainId), "domain_id")
-		}
-		rid, err := modules.RolesV3.GetId(s, args.ID, query)
-		if err != nil {
-			return err
-		}
-		role, err := modules.RolesV3.Delete(s, rid, nil)
-		if err != nil {
-			return err
-		}
-		printObject(role)
-		return nil
-	})
-
-	type RoleCreateOptions struct {
-		NAME   string `help:"Role name"`
-		Domain string `help:"Domain"`
-		Desc   string `help:"Description"`
-
-		PublicScope string `help:"public scope" choices:"none|system"`
-	}
-	R(&RoleCreateOptions{}, "role-create", "Create a new role", func(s *mcclient.ClientSession, args *RoleCreateOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.NAME), "name")
-		if len(args.Domain) > 0 {
-			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
-			if err != nil {
-				return err
-			}
-			params.Add(jsonutils.NewString(domainId), "domain_id")
-		}
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
-		}
-		if len(args.PublicScope) > 0 {
-			params.Add(jsonutils.NewString(args.PublicScope), "public_scope")
-		}
-		role, err := modules.RolesV3.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(role)
-		return nil
-	})
+	cmd := shell.NewResourceCmd(&modules.RolesV3)
+	cmd.List(&identity.RoleListOptions{})
+	cmd.Show(&identity.RoleDetailOptions{})
+	cmd.Delete(&identity.RoleDetailOptions{})
+	cmd.Create(&identity.RoleCreateOptions{})
+	cmd.Perform("public", &options.BaseIdOptions{})
+	cmd.Perform("private", &options.BaseIdOptions{})
 
 	type RoleUpdateOptions struct {
 		ID     string `help:"Role ID or Name"`
@@ -145,25 +66,4 @@ func init() {
 		printObject(role)
 		return nil
 	})
-
-	type RolePerformOptions struct {
-		ID string `help:"ID of role to update"`
-	}
-	R(&RolePerformOptions{}, "role-public", "Mark a role public", func(s *mcclient.ClientSession, args *RolePerformOptions) error {
-		result, err := modules.RolesV3.PerformAction(s, args.ID, "public", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&RolePerformOptions{}, "role-private", "Mark a role private", func(s *mcclient.ClientSession, args *RolePerformOptions) error {
-		result, err := modules.RolesV3.PerformAction(s, args.ID, "private", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
 }

cmd/climc/shell/image/images.go

@@ -142,13 +142,15 @@ func init() {
 	type ImageListOptions struct {
 		options.BaseListOptions
 
-		IsPublic   string   `help:"filter images public or not(True, False or None)" choices:"true|false"`
-		IsStandard string   `help:"filter images standard or non-standard" choices:"true|false"`
-		Protected  string   `help:"filter images by protected" choices:"true|false"`
-		IsUefi     bool     `help:"list uefi image"`
-		Format     []string `help:"Disk formats"`
-		SubFormats []string `help:"Sub formats"`
-		Name       string   `help:"Name filter"`
+		IsPublic     string   `help:"filter images public or not(True, False or None)" choices:"true|false"`
+		IsStandard   string   `help:"filter images standard or non-standard" choices:"true|false"`
+		Protected    string   `help:"filter images by protected" choices:"true|false"`
+		IsUefi       bool     `help:"list uefi image"`
+		Format       []string `help:"Disk formats"`
+		SubFormats   []string `help:"Sub formats"`
+		Name         string   `help:"Name filter"`
+		OsType       []string `help:"Type of OS filter e.g. 'Windows, Linux, Freebsd, Android, macOS, VMWare'"`
+		Distribution []string `help:"Distribution filter, e.g. 'CentOS, Ubuntu, Debian, Windows'"`
 	}
 	R(&ImageListOptions{}, "image-list", "List images", func(s *mcclient.ClientSession, args *ImageListOptions) error {
 		params, err := args.Params()
@@ -187,6 +189,12 @@ func init() {
 		if len(args.SubFormats) > 0 {
 			params.Add(jsonutils.Marshal(args.SubFormats), "sub_formats")
 		}
+		if len(args.OsType) > 0 {
+			params.Add(jsonutils.NewStringArray(args.OsType), "os_types")
+		}
+		if len(args.Distribution) > 0 {
+			params.Add(jsonutils.NewStringArray(args.Distribution), "distributions")
+		}
 		result, err := modules.Images.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/k8s/clusters.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
 	o "yunion.io/x/onecloud/pkg/mcclient/options/k8s"
@@ -201,18 +202,32 @@ func initKubeCluster() {
 		return nil
 	})
 
-	R(&o.ClusterEnableComponentMinioOpt{}, cmdN("component-enable-minio"), "Enable cluster minio component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentMinioOpt) error {
-		params, err := args.Params()
-		if err != nil {
-			return err
-		}
-		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "enable-component", params)
-		if err != nil {
-			return err
-		}
-		printObject(ret)
-		return nil
-	})
+	rEnableMinio := func(opt shell.IPerformOpt, cmd string, desc string) {
+		R(opt, cmd, desc, func(s *mcclient.ClientSession, args shell.IPerformOpt) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.KubeClusters.PerformAction(s, args.GetId(), "enable-component", params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+	}
+
+	rEnableMinio(
+		new(o.ClusterEnableComponentMinioOpt),
+		cmdN("component-enable-minio"),
+		"Enable cluster minio component",
+	)
+
+	rEnableMinio(
+		new(o.ClusterEnableComponentMonitorMinioOpt),
+		cmdN("component-enable-monitor-minio"),
+		"Enable cluster monitor stack's minio component",
+	)
 
 	R(&o.ClusterEnableComponentThanosOpt{}, cmdN("component-enable-thanos"), "Enable cluster thanos component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentThanosOpt) error {
 		params, err := args.Params()
@@ -260,9 +275,14 @@ func initKubeCluster() {
 		return nil
 	})
 
-	R(&o.ClusterComponentType{}, cmdN("component-update"), "Update cluster component", func(s *mcclient.ClientSession, args *o.ClusterComponentType) error {
+	type ClusterComponentTypeUpdate struct {
+		o.ClusterComponentType
+		Force bool `help:"Force update"`
+	}
+
+	R(&ClusterComponentTypeUpdate{}, cmdN("component-update"), "Update cluster component", func(s *mcclient.ClientSession, args *ClusterComponentTypeUpdate) error {
 		// 1. get current setting
-		setting, err := getComponentSetting(s, args, false)
+		setting, err := getComponentSetting(s, &args.ClusterComponentType, false)
 		if err != nil {
 			return errors.Wrap(err, "get component setting")
 		}
@@ -270,8 +290,11 @@ func initKubeCluster() {
 		// 2. edit yaml
 		yaml, err := shellutils.Edit(setting.YAMLString())
 		if len(yaml) == 0 {
-			log.Infof("Nothing to update")
-			return nil
+			if !args.Force {
+				log.Infof("Nothing to update")
+				return nil
+			}
+			yaml = setting.YAMLString()
 		}
 		nowSetting, err := jsonutils.ParseYAML(yaml)
 		if err != nil {
@@ -279,6 +302,7 @@ func initKubeCluster() {
 		}
 		params := args.Params(args.TYPE)
 		params.Update(nowSetting)
+		params.Add(jsonutils.NewBool(args.Force), "force")
 
 		// 3. call update api
 		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "update-component", params)

cmd/climc/shell/meter/bills_dimension.go

@@ -0,0 +1,20 @@
+package meter
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/meter"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(modules.BillingDimensionManager)
+	cmd.Create(&options.BillDimensionCreateOptions{})
+	cmd.List(&options.BillDimensionListOptions{})
+	cmd.Show(&options.BillDimensionShowOptions{})
+	cmd.Delete(&options.BillDimensionDeleteOptions{})
+
+	cmdAnalysis := shell.NewResourceCmd(modules.BillingDimensionAnalysisManager)
+	cmdAnalysis.List(&options.BillDimensionAnalysisListOptions{})
+	cmdAnalysis.Show(&options.BillDimensionAnalysisShowOptions{})
+
+}

cmd/climc/shell/meter/cost_conversions.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package meter
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/meter"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.CostConversions).WithKeyword("cost-conversion")
+	cmd.List(new(options.CostConversionListOptions))
+	cmd.Create(new(options.CostConversionCreateOptions))
+	cmd.Update(new(options.CostConversionUpdateOptions))
+	cmd.Delete(new(options.CostConversionDeleteOptions))
+}

cmd/climc/shell/misc/licenses.go

@@ -15,6 +15,12 @@
 package misc
 
 import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -40,6 +46,40 @@ func init() {
 		return nil
 	})
 
+	type LicenseUploadOptions struct {
+		FILE string
+	}
+
+	R(&LicenseUploadOptions{}, "license-upload", "Upload license", func(s *mcclient.ClientSession, args *LicenseUploadOptions) error {
+		bodyBuf := &bytes.Buffer{}
+		bodyWriter := multipart.NewWriter(bodyBuf)
+		fileWriter, err := bodyWriter.CreateFormFile("license", args.FILE)
+		if err != nil {
+			return err
+		}
+		fh, err := os.Open(args.FILE)
+		if err != nil {
+			return err
+		}
+		defer fh.Close()
+		_, err = io.Copy(fileWriter, fh)
+		if err != nil {
+			return err
+		}
+		contentType := bodyWriter.FormDataContentType()
+		bodyWriter.Close()
+
+		header := http.Header{}
+		header.Set("Content-Type", contentType)
+		lic, err := modules.License.Upload(s, header, bodyBuf)
+		if err != nil {
+			return err
+		}
+
+		printObject(lic)
+		return nil
+	})
+
 	type LicenseShowOptions struct {
 		SERVICE string `help:"service name"  choices:"compute|service_tree"`
 	}

cmd/climc/shell/misc/version.go

@@ -17,6 +17,11 @@ package misc
 import (
 	"fmt"
 
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -35,6 +40,43 @@ func init() {
 		return nil
 	})
 
+	type VersionListOptions struct {
+	}
+
+	R(&VersionListOptions{}, "version-list", "query all backend service version", func(s *mcclient.ClientSession, args *VersionListOptions) error {
+		services := []jsonutils.JSONObject{}
+		params := map[string]interface{}{}
+		for {
+			params["offset"] = len(services)
+			resp, err := modules.ServicesV3.List(s, jsonutils.Marshal(params))
+			if err != nil {
+				return err
+			}
+			services = append(services, resp.Data...)
+			if len(services) >= resp.Total {
+				break
+			}
+		}
+		vers := map[string]string{}
+		for _, service := range services {
+			serviceType, _ := service.GetString("type")
+			if utils.IsInStringArray(serviceType, []string{"offlinecloudmeta"}) {
+				continue
+			}
+			ver, err := modules.GetVersion(s, serviceType)
+			if err != nil {
+				if errors.Cause(err) == cloudprovider.ErrNotFound || errors.Cause(err) == errors.ErrConnectRefused {
+					continue
+				}
+				vers[serviceType] = err.Error()
+			} else {
+				vers[serviceType] = ver
+			}
+		}
+		printObject(jsonutils.Marshal(vers))
+		return nil
+	})
+
 	R(&options.VersionListOptions{}, "yunionagent-version-list", "show versions of backend services", func(s *mcclient.ClientSession, opts *options.VersionListOptions) error {
 		if len(opts.Region) == 0 {
 			opts.Region = s.GetRegion()

cmd/climc/shell/monitor/monitor_resource.go

@@ -0,0 +1,27 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monitor
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(modules.MonitorResourceManager)
+	cmd.Get("", new(options.MonitorResourceJointAlertOptions))
+
+}

cmd/cloudmon/main.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/version"
+
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) {
+	if len(options.AuthURL) == 0 {
+		return nil, errors.Error("empty auth_url")
+	}
+	if len(options.AdminUser) == 0 {
+		return nil, errors.Error("empty admin_user")
+	}
+	if len(options.AdminPassword) == 0 {
+		return nil, errors.Error("empty admin_password")
+	}
+	if len(options.AdminProject) == 0 {
+		return nil, errors.Error("empty admin_project")
+	}
+
+	client := mcclient.NewClient(
+		options.AuthURL,
+		options.Timeout,
+		options.Debug,
+		options.Insecure,
+		options.CertFile,
+		options.KeyFile,
+	)
+
+	token, err := client.AuthenticateWithSource(
+		options.AdminUser,
+		options.AdminPassword,
+		options.AdminDomain,
+		options.AdminProject,
+		options.AdminProjectDomain,
+		mcclient.AuthSourceAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	session := client.NewSession(
+		context.Background(),
+		options.Region,
+		"",
+		options.EndpointType,
+		token,
+		options.ApiVersion)
+
+	return session, nil
+}
+
+func main() {
+	parser, err := options.GetArgumentParser()
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	err = parser.ParseArgs2(os.Args[1:], false, false)
+
+	opts := parser.Options().(*options.CloudMonOptions)
+
+	if opts.Help {
+		fmt.Println(parser.HelpString())
+		os.Exit(0)
+	}
+
+	if opts.Version {
+		fmt.Println(version.GetJsonString())
+		os.Exit(0)
+	}
+
+	if len(opts.Config) == 0 {
+		for _, p := range []string{"./etc", "/etc/yunion"} {
+			confTmp := path.Join(p, "cloudmon.conf")
+			if _, err := os.Stat(confTmp); err == nil {
+				opts.Config = confTmp
+				break
+			}
+		}
+	}
+
+	if len(opts.Config) > 0 {
+		err := parser.ParseFile(opts.Config)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+	}
+
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	parser.SetDefault()
+
+	subcmd := parser.GetSubcommand()
+	subparser := subcmd.GetSubParser()
+	if err != nil {
+		if subparser != nil {
+			fmt.Print(subparser.Usage())
+		} else {
+			fmt.Print(parser.Usage())
+		}
+		showErrorAndExit(err)
+	}
+
+	suboptions := subparser.Options()
+	if opts.SUBCOMMAND == "help" {
+		err = subcmd.Invoke(suboptions)
+	} else {
+		timout := suboptions.(*common.ReportOptions).Timeout
+		endChan := make(chan int, 1)
+		go func() {
+			ticker := time.NewTicker(time.Duration(timout) * time.Second)
+			for {
+				select {
+				case <-ticker.C:
+					log.Errorf("cmd: %s,provider: %v,end due to timeout: %ds", opts.SUBCOMMAND,
+						suboptions.(*common.ReportOptions).Provider, suboptions.(*common.ReportOptions).Timeout)
+					os.Exit(3)
+				case <-endChan:
+				}
+			}
+		}()
+
+		var session *mcclient.ClientSession
+		session, err = newClientSession(opts)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+		err = subcmd.Invoke(session, suboptions)
+		endChan <- 1
+	}
+	if err != nil {
+		showErrorAndExit(err)
+	}
+}

cmd/cryptool/shell/fernet.go

@@ -16,7 +16,6 @@ package shell
 
 import (
 	"fmt"
-	"time"
 
 	"yunion.io/x/pkg/util/timeutils"
 
@@ -110,7 +109,7 @@ func init() {
 		}
 		fmt.Println("primary key hash:", fm.PrimaryKeyHash())
 
-		ret := fm.Decrypt([]byte(args.MSG), time.Hour*-1)
+		ret := fm.Decrypt([]byte(args.MSG))
 		if len(ret) == 0 {
 			return fmt.Errorf("invalid message")
 		}

cmd/hcsocli/main.go

@@ -0,0 +1,147 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	huawei "yunion.io/x/onecloud/pkg/multicloud/hcso"
+	_ "yunion.io/x/onecloud/pkg/multicloud/hcso/shell"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BaseOptions struct {
+	cloudprovider.SHCSOEndpoints
+	Help       bool   `help:"Show help" default:"false"`
+	Debug      bool   `help:"Show debug" default:"false"`
+	AccessKey  string `help:"Access key" default:"$HUAWEI_ACCESS_KEY" metavar:"HUAWEI_ACCESS_KEY"`
+	Secret     string `help:"Secret" default:"$HUAWEI_SECRET" metavar:"HUAWEI_SECRET"`
+	RegionId   string `help:"RegionId" default:"$HUAWEI_REGION" metavar:"HUAWEI_REGION"`
+	ProjectId  string `help:"ProjectId" default:"$HUAWEI_PROJECT" metavar:"HUAWEI_PROJECT"`
+	SUBCOMMAND string `help:"huaweicli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"huaweicli",
+		"Command-line interface to huawei API.",
+		`See "huaweicli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*huawei.SRegion, error) {
+	if len(options.AccessKey) == 0 {
+		return nil, fmt.Errorf("Missing accessKey")
+	}
+
+	if len(options.Secret) == 0 {
+		return nil, fmt.Errorf("Missing secret")
+	}
+
+	cli, err := huawei.NewHuaweiClient(
+		huawei.NewHuaweiClientConfig(
+			options.AccessKey,
+			options.Secret,
+			options.ProjectId,
+			&options.SHCSOEndpoints,
+		).Debug(options.Debug),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	region := cli.GetRegion(options.RegionId)
+	if region == nil {
+		return nil, fmt.Errorf("No such region %s", options.RegionId)
+	}
+
+	return region, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var region *huawei.SRegion
+				region, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(region, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/telegraf-raid-plugin/main.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 const (
@@ -128,6 +129,7 @@ func (c *RaidInfoCollector) CollectReportData() string {
 }
 
 func (c *RaidInfoCollector) Start() {
+	go procutils.WaitZombieLoop(context.TODO())
 	for {
 		c.runMain()
 		time.Sleep(time.Second * 1)

cmd/webconsole/main.go

@@ -15,12 +15,17 @@
 package main
 
 import (
+	"context"
+
 	"yunion.io/x/onecloud/pkg/util/atexit"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/webconsole/service"
 )
 
 func main() {
 	defer atexit.Handle()
 
+	go procutils.WaitZombieLoop(context.TODO())
+
 	service.StartService()
 }

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/coredns/coredns v1.3.0
 	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/creack/pty v1.1.11
 	github.com/dnaeon/go-vcr v1.1.0 // indirect
 	github.com/dnstap/golang-dnstap v0.0.0-20170829151710-2cf77a2b5e11 // indirect
 	github.com/eapache/go-resiliency v1.1.0 // indirect
@@ -54,7 +55,6 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.5.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
-	github.com/go-ini/ini v1.62.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
@@ -77,7 +77,6 @@ require (
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
-	github.com/kr/pty v1.1.5
 	github.com/lestrrat-go/jwx v1.0.2
 	github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae
 	github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect
@@ -113,6 +112,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
+	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect
@@ -146,13 +146,13 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
-	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
+	yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
+	yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c
+	yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824
+	yunion.io/x/sqlchemy v1.0.0
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -2,14 +2,12 @@ bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVc
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0 h1:0E3eE8MX426vUOs7aHfI7aN1BrIzzzf4ccKCSfSjGmc=
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM=
 cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
@@ -32,16 +30,12 @@ github.com/Azure/azure-sdk-for-go v36.1.0+incompatible h1:smHlbChr/JDmsyUqELZXLs
 github.com/Azure/azure-sdk-for-go v36.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.3 h1:OZEIaBbMdUE/Js+BQKlpO81XlISgipr6yDJ+PSwsgi4=
 github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0=
 github.com/Azure/go-autorest/autorest v0.9.6 h1:5YWtOnckcudzIw8lPPBcWOnmIFWMtHci1ZWAZulMSx0=
 github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
-github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
-github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk=
 github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0=
 github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
@@ -49,12 +43,10 @@ github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwp
 github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM=
 github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U=
 github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw=
-github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
 github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
 github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
 github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
@@ -180,6 +172,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -241,8 +235,6 @@ github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
-github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
@@ -405,8 +397,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
@@ -586,6 +576,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:nt2877sKfojlHCTOBXbpWjBkuWKritFaGIfgQwbQUls=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:B4+Kq1u5FlULTjFSM707Q6e/cOHFv0z/6QRoxubDIQ8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=
@@ -701,10 +693,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -750,7 +740,6 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -910,11 +899,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a h1:X/ucfLNcYsCfc4GFEX7R7GNWiDPr3vopLdnE4PuU46U=
+yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o=
@@ -924,11 +913,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c h1:J/joqbA1N2mAlOl0Uqd4LpAq3+DK5aoFMdz+p9Ld7pQ=
-yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 h1:3/VcCiO+vr3+zZFLNvre41ARF2xE6NVq6Ue7JPFuJvU=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v1.0.0 h1:fRtWxe1xxMR+NDVKYNuI89m2HGSl6aad2DZ+Bn/mZ4Q=
+yunion.io/x/sqlchemy v1.0.0/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/ansibleserver/models/ansibleplaybook_instance.go

@@ -138,26 +138,17 @@ func (ai *SAnsiblePlaybookInstance) runPlaybook(ctx context.Context, userCred mc
 		return errors.Wrap(err, "unable to update ansibleplaybookinstance")
 	}
 
-	convertJO := func(o jsonutils.JSONObject) map[string]interface{} {
-		if o == nil {
-			return map[string]interface{}{}
-		}
-		ret := make(map[string]interface{})
-		o.Unmarshal(&ret)
-		return ret
-	}
 	// merge configs
-	params := make(map[string]interface{})
-	for k, v := range convertJO(ar.DefaultParams) {
-		params[k] = v
-	}
-	for k, v := range convertJO(ai.Params) {
-		params[k] = v
+	dp, params := ar.DefaultParams.(*jsonutils.JSONDict), ai.Params.(*jsonutils.JSONDict)
+	for _, k := range dp.SortedKeys() {
+		v, _ := dp.Get(k)
+		params.Set(k, v)
 	}
+	ar.DefaultParams.(*jsonutils.JSONDict).SortedKeys()
 	sess := ansiblev2.NewOfflineSession().
 		Inventory(ai.Inventory).
 		PrivateKey(privateKey).
-		Configs(params).
+		ConfigYaml(params.YAMLString()).
 		PlaybookPath(ar.PlaybookPath).
 		OutputWriter(&ansiblePlaybookOutputWriter{ai}).
 		KeepTmpdir(options.Options.KeepTmpdir)

pkg/ansibleserver/models/ansibleplaybooks.go

@@ -121,7 +121,7 @@ func (man *SAnsiblePlaybookManager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/ansibleserver/models/ansibleplaybooks_v2.go

@@ -43,7 +43,7 @@ type SAnsiblePlaybookV2 struct {
 
 	Playbook     string    `length:"text" nullable:"false" create:"required" get:"user"`
 	Inventory    string    `length:"text" nullable:"false" create:"required" get:"user"`
-	Requirements string    `length:"test" nullable:"false" create:"optional" get:"user"`
+	Requirements string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Files        string    `length:"text" nullable:"false" create:"optional" get:"user"`
 	Output       string    `length:"medium" get:"user"`
 	StartTime    time.Time `list:"user"`
@@ -118,7 +118,7 @@ func (man *SAnsiblePlaybookV2Manager) InitializeData() error {
 	return nil
 }
 
-func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context) error {
+func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
 	if apb.Status == api.AnsiblePlaybookStatusRunning {
 		return httperrors.NewConflictError("playbook is in running state")
 	}

pkg/apigateway/clientman/authtoken.go

@@ -275,7 +275,7 @@ func (t *SAuthToken) VerifyTotpPasscode(s *mcclient.ClientSession, uid, passcode
 	}
 
 	t.updateRetryCount()
-	return errors.Wrap(httperrors.ErrInvalidCredential, "invalid passcode")
+	return errors.Wrap(httperrors.ErrInputParameter, "invalid passcode")
 }
 
 func SignJWT(t jwt.Token) (string, error) {

pkg/apigateway/handler/auth.go

@@ -185,7 +185,22 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 
 	resp.Add(jsonutils.NewArray(retIdps...), "idps")
 
-	resp.Add(jsonutils.NewString(options.Options.ApiServer), "api_server")
+	// in case api_server changed but not synchronized from Keystone
+	// fetch this option directly from Keystone
+	var apiSrv string
+	commonCfg, err := modules.ServicesV3.GetSpecific(s, "common", "config", nil)
+	if err != nil {
+		log.Errorf("fetch common config error: %s", err)
+	} else {
+		apiSrv, err = commonCfg.GetString("config", "default", "api_server")
+		if err != nil {
+			log.Errorf("fetch api_server fail: %s (cfg: %s)", err, commonCfg)
+		}
+	}
+	if len(apiSrv) == 0 {
+		apiSrv = options.Options.ApiServer
+	}
+	resp.Add(jsonutils.NewString(apiSrv), "api_server")
 
 	return resp, nil
 }
@@ -1183,16 +1198,27 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 		return
 	}
 
-	oldPwd, _ := body.GetString("password_old")
-	newPwd, _ := body.GetString("password_new")
-	confirmPwd, _ := body.GetString("password_confirm")
-	passcode, _ := body.GetString("passcode")
+	input := struct {
+		PasswordOld     string
+		PasswordNew     string
+		PasswordConfirm string
+		Passcode        string
+	}{}
+	body.Unmarshal(&input)
 
-	if newPwd != confirmPwd {
+	if input.PasswordNew != input.PasswordConfirm {
 		httperrors.InputParameterError(ctx, w, "new password mismatch")
 		return
 	}
 
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordOld); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordOld = string(decPasswd)
+	}
+
+	if decPasswd, err := base64.StdEncoding.DecodeString(input.PasswordNew); err == nil && stringutils2.IsPrintableAsciiString(string(decPasswd)) {
+		input.PasswordNew = string(decPasswd)
+	}
+
 	// 1.验证原密码正确，且idp_driver为空
 	if isIdpUser(user) {
 		httperrors.ForbiddenError(ctx, w, "not support reset user password")
@@ -1200,7 +1226,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	}
 
 	cliIp := netutils2.GetHttpRequestIp(req)
-	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), oldPwd, t.GetDomainName(), "", "", cliIp)
+	_, err = auth.Client().AuthenticateWeb(t.GetUserName(), input.PasswordOld, t.GetDomainName(), "", "", cliIp)
 	if err != nil {
 		switch httperr := err.(type) {
 		case *httputils.JSONClientError:
@@ -1216,7 +1242,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	// 2.如果已开启MFA，验证 随机密码正确
 	if isMfaEnabled(user) {
-		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), passcode)
+		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), input.Passcode)
 		if err != nil {
 			httperrors.InputParameterError(ctx, w, "invalid passcode")
 			return
@@ -1225,7 +1251,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 
 	// 3.重置密码，
 	params := jsonutils.NewDict()
-	params.Set("password", jsonutils.NewString(newPwd))
+	params.Set("password", jsonutils.NewString(input.PasswordNew))
 	_, err = modules.UsersV3.Patch(s, t.GetUserId(), params)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)

pkg/apigateway/handler/auth_totp.go

@@ -260,7 +260,7 @@ func validatePasscodeHandler(ctx context.Context, w http.ResponseWriter, req *ht
 
 	if err != nil {
 		log.Warningf("VerifyTotpPasscode %s", err.Error())
-		httperrors.InvalidCredentialError(ctx, w, "invalid passcode: %v", err)
+		httperrors.InputParameterError(ctx, w, "invalid passcode: %v", err)
 		return
 	}
 

pkg/apigateway/handler/dispatcher.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"time"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
@@ -51,7 +52,17 @@ func (h SHandler) Bind(app *appsrv.Application) {
 	if h.MiddlewareFunc != nil {
 		f = h.MiddlewareFunc(f)
 	}
-	app.AddHandler(h.Method, h.Prefix, f)
+	segs := appsrv.SplitPath(h.Prefix)
+	hi := appsrv.NewHandlerInfo(h.Method, segs, f, nil, "", nil)
+	if h.Method == "GET" {
+		hi.SetProcessTimeoutCallback(func(hdr *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+			if len(r.URL.Query().Get("export_keys")) > 0 {
+				return time.Hour * 2
+			}
+			return -time.Second
+		})
+	}
+	app.AddHandler3(hi)
 }
 
 func NewMethodHandlerFactory(method string, mf appsrv.MiddlewareFunc, prefix string) func(handleFunc, ...string) SHandler {

pkg/apigateway/handler/idp.go

@@ -116,7 +116,7 @@ func (h *AuthHandlers) getIdpSsoRedirectUri(ctx context.Context, w http.Response
 }
 
 func findExtUserId(input string) string {
-	pattern := regexp.MustCompile(`idp.SyncOrCreateDomainAndUser: ([^:]+): UserNotFoundError`)
+	pattern := regexp.MustCompile(`idp.SyncOrCreateDomainAndUser: ([^:]+): UserNotFound`)
 	matches := pattern.FindAllStringSubmatch(input, -1)
 	log.Debugf("%#v", matches)
 	if len(matches) > 0 && len(matches[0]) > 1 {

pkg/apigateway/handler/idp_test.go

@@ -22,7 +22,7 @@ func TestFindExtUserId(t *testing.T) {
 		want string
 	}{
 		{
-			in:   "authUserByCASV3: Authenticate: idp.SyncOrCreateDomainAndUser: qiujian: UserNotFoundError",
+			in:   "authUserByCASV3: Authenticate: idp.SyncOrCreateDomainAndUser: qiujian: UserNotFound",
 			want: "qiujian",
 		},
 	}

pkg/apigateway/handler/oidc.go

@@ -100,6 +100,7 @@ func handleOIDCAuth(ctx context.Context, w http.ResponseWriter, req *http.Reques
 	qs.Set("code", jsonutils.NewString(code))
 	qs.Set("state", jsonutils.NewString(auth.State))
 	redirUrl := addQuery(auth.RedirectUri, qs)
+	appsrv.DisableClientCache(w)
 	appsrv.SendRedirect(w, redirUrl)
 }
 

pkg/apigateway/handler/resource.go

@@ -22,6 +22,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -190,45 +191,62 @@ func (f *ResourceHandlers) listHandler(ctx context.Context, w http.ResponseWrite
 func (f *ResourceHandlers) doList(ctx context.Context, session *mcclient.ClientSession, module modulebase.IBaseManager, query jsonutils.JSONObject, w http.ResponseWriter, r *http.Request) {
 	var exportKeys []string
 	var exportTexts []string
-	exportFormat, _ := query.GetString("export")
-	if len(exportFormat) > 0 {
-		exportKeyStr, _ := query.GetString("export_keys")
-		exportTextStr, _ := query.GetString("export_texts")
-		if len(exportKeyStr) > 0 {
-			exportKeys = strings.Split(exportKeyStr, ",")
+	export := struct {
+		ExportFormat string `json:"export"`
+		ExportKeys   string `json:"export_keys"`
+		ExportTexts  string `json:"export_texts"`
+	}{}
+	query.Unmarshal(&export)
+	if len(export.ExportFormat) > 0 {
+		if len(export.ExportKeys) > 0 {
+			exportKeys = strings.Split(export.ExportKeys, ",")
 		}
-		if len(exportTextStr) > 0 {
-			exportTexts = strings.Split(exportTextStr, ",")
+		if len(export.ExportTexts) > 0 {
+			exportTexts = strings.Split(export.ExportTexts, ",")
 		}
 		if len(exportKeys) == 0 {
 			httperrors.InvalidInputError(ctx, w, "missing export keys")
 			return
-		} else if len(exportKeys) != len(exportTexts) {
-			if len(exportTexts) == 0 {
-				exportTexts = exportKeys
-			} else {
-				httperrors.InvalidInputError(ctx, w, "inconsistent export keys and texts")
-				return
-			}
+		}
+		if len(exportTexts) == 0 {
+			exportTexts = exportKeys
+		}
+		if len(exportKeys) != len(exportTexts) {
+			httperrors.InvalidInputError(ctx, w, "inconsistent export keys and texts")
+			return
 		}
 		queryDict := query.(*jsonutils.JSONDict)
 		queryDict.Remove("export")
 		queryDict.Remove("export_texts")
+		keys := []string{}
+		for _, key := range exportKeys {
+			if !strings.Contains(key, ".") {
+				keys = append(keys, key)
+				continue
+			}
+			key = strings.Split(key, ".")[0]
+			if !utils.IsInStringArray(key, keys) {
+				keys = append(keys, key)
+			}
+		}
+		queryDict.Set("export_keys", jsonutils.NewString(strings.Join(keys, ",")))
 		query = queryDict
 	}
 
 	ret, e := module.List(session, query)
 	if e != nil {
 		httperrors.GeneralServerError(ctx, w, e)
-	} else if len(exportFormat) > 0 {
+		return
+	}
+	if len(export.ExportFormat) > 0 {
 		w.Header().Set("Content-Description", "File Transfer")
 		w.Header().Set("Content-Transfer-Encoding", "binary")
 		w.Header().Set("Content-Type", "application/octet-stream")
 		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"export-%s.xlsx\"", module.KeyString()))
 		excelutils.Export(ret.Data, exportKeys, exportTexts, w)
-	} else {
-		appsrv.SendJSON(w, modulebase.ListResult2JSON(ret))
+		return
 	}
+	appsrv.SendJSON(w, modulebase.ListResult2JSON(ret))
 }
 
 // get single
@@ -787,7 +805,7 @@ func (f *ResourceHandlers) detachHandle(ctx context.Context, w http.ResponseWrit
 	jmod, e := modulebase.GetJointModule2(session, module, module2)
 	var obj jsonutils.JSONObject
 	if e == nil { // joint detach
-		obj, e = jmod.Detach(session, req.ResID(), req.ResID2(), nil)
+		obj, e = jmod.Detach(session, req.ResID(), req.ResID2(), req.Query())
 	} else {
 		obj, e = module2.DeleteInContextWithParam(session, req.ResID2(), req.Query(), req.Body(), module, req.ResID())
 	}

pkg/apihelper/apihelper.go

@@ -101,7 +101,7 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) {
 
 	s := h.adminClientSession(ctx)
 	mss := h.modelSets.Copy()
-	r, err := SyncModelSets(mss, s, h.opts.ListBatchSize)
+	r, err := SyncModelSets(mss, s, h.opts)
 	if err != nil {
 		return false, err
 	}
@@ -118,7 +118,7 @@ func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess
 	if s != nil {
 		token := s.GetToken()
 		expires := token.GetExpires()
-		if time.Now().Add(time.Hour).After(expires) {
+		if time.Now().Add(time.Hour).Before(expires) {
 			return s
 		}
 	}

pkg/apihelper/interface.go

@@ -15,6 +15,8 @@
 package apihelper
 
 import (
+	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	mcclient "yunion.io/x/onecloud/pkg/mcclient"
 	mcclient_modulebase "yunion.io/x/onecloud/pkg/mcclient/modulebase"
@@ -44,7 +46,15 @@ type IModelSetEmulatedIncluder interface {
 	IncludeEmulated() bool
 }
 
-func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int) (r ModelSetsUpdateResult, err error) {
+type IModelSetFilter interface {
+	ModelFilter() []string
+}
+
+type IModelListParam interface {
+	ModelParamFilter() jsonutils.JSONObject
+}
+
+func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) {
 	mss := mssOld.ModelSetList()
 	mssNews := mssOld.NewEmpty()
 	for i, msNew := range mssNews.ModelSetList() {
@@ -60,10 +70,11 @@ func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, batchSize int)
 			ModelManager:  msNew.ModelManager(),
 			MinUpdatedAt:  minUpdatedAt,
 			ModelSet:      msNew,
-			BatchListSize: batchSize,
+			BatchListSize: opt.ListBatchSize,
 
-			IncludeDetails:  false,
-			IncludeEmulated: includeEmulated,
+			IncludeDetails:       opt.IncludeDetails,
+			IncludeEmulated:      includeEmulated,
+			InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv,
 		})
 		if err != nil {
 			return

pkg/apihelper/options.go

@@ -21,6 +21,8 @@ import (
 type Options struct {
 	common_options.CommonOptions
 
-	SyncInterval  int
-	ListBatchSize int
+	SyncInterval         int
+	ListBatchSize        int
+	IncludeDetails       bool
+	IncludeOtherCloudEnv bool
 }

pkg/apihelper/reflect.go

@@ -38,10 +38,11 @@ type GetModelsOptions struct {
 	ModelManager  mcclient_modulebase.IBaseManager
 	ModelSet      IModelSet
 
-	BatchListSize   int
-	MinUpdatedAt    time.Time
-	IncludeDetails  bool
-	IncludeEmulated bool
+	BatchListSize        int
+	MinUpdatedAt         time.Time
+	IncludeDetails       bool
+	IncludeEmulated      bool
+	InCludeOtherCloudEnv bool
 }
 
 func GetModels(opts *GetModelsOptions) error {
@@ -102,19 +103,28 @@ func GetModels(opts *GetModelsOptions) error {
 
 	listOptions := options.BaseListOptions{
 		Admin:        options.Bool(true),
+		Scope:        "system",
 		Details:      options.Bool(opts.IncludeDetails),
 		ShowEmulated: options.Bool(opts.IncludeEmulated),
 		Filter: []string{
 			minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0
-			"manager_id.isnullorempty()",     // len(manager_id) > 0 is for pubcloud objects
-			"external_id.isnullorempty()",    // len(external_id) > 0 is for pubcloud objects
-			"cloud_env=onpremise",
 		},
 		OrderBy: []string{"updated_at"},
 		Order:   "asc",
 		Limit:   options.Int(opts.BatchListSize),
 		Offset:  options.Int(0),
 	}
+	if !opts.InCludeOtherCloudEnv {
+		listOptions.Filter = append(listOptions.Filter,
+			"manager_id.isnullorempty()",  // len(manager_id) > 0 is for pubcloud objects
+			"external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
+			"cloud_env=onpremise",
+		)
+	}
+	if inter, ok := opts.ModelSet.(IModelSetFilter); ok {
+		filter := inter.ModelFilter()
+		listOptions.Filter = append(listOptions.Filter, filter...)
+	}
 	if !minUpdatedAt.Equal(PseudoZeroTime) {
 		// Only fetching pending deletes when we are doing incremental fetch
 		listOptions.PendingDeleteAll = options.Bool(true)
@@ -124,6 +134,10 @@ func GetModels(opts *GetModelsOptions) error {
 	if err != nil {
 		return fmt.Errorf("%s: making list params: %s", manKeyPlural, err)
 	}
+	if inter, ok := opts.ModelSet.(IModelListParam); ok {
+		filter := inter.ModelParamFilter()
+		params.Update(filter)
+	}
 	//XXX
 	//params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL))
 

pkg/apis/compute/api.go

@@ -396,6 +396,10 @@ type ServerCreateInput struct {
 	// emulate: BIOS, UEFI
 	Bios string `json:"bios"`
 
+	// Machine类型
+	// emulate: pc, q35
+	Machine string `json:"machine"`
+
 	// 启动顺序
 	// c: cdrome
 	// d: disk

pkg/apis/compute/baremetal_const.go

@@ -29,12 +29,13 @@ const (
 
 	DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE
 
-	DISK_DRIVER_MEGARAID   = "MegaRaid"
-	DISK_DRIVER_LINUX      = "Linux"
-	DISK_DRIVER_HPSARAID   = "HPSARaid"
-	DISK_DRIVER_MPT2SAS    = "Mpt2SAS"
-	DISK_DRIVER_MARVELRAID = "MarvelRaid"
-	DISK_DRIVER_PCIE       = "PCIE"
+	DISK_DRIVER_MEGARAID    = "MegaRaid"
+	DISK_DRIVER_LINUX       = "Linux"
+	DISK_DRIVER_HPSARAID    = "HPSARaid"
+	DISK_DRIVER_MPT2SAS     = "Mpt2SAS"
+	DISK_DRIVER_MARVELRAID  = "MarvelRaid"
+	DISK_DRIVER_ADAPTECRAID = "AdaptecRaid"
+	DISK_DRIVER_PCIE        = "PCIE"
 
 	HDD_DISK_SPEC_TYPE = "HDD"
 	SSD_DISK_SPEC_TYPE = "SSD"
@@ -60,6 +61,7 @@ var (
 		DISK_DRIVER_HPSARAID,
 		DISK_DRIVER_MPT2SAS,
 		DISK_DRIVER_MARVELRAID,
+		DISK_DRIVER_ADAPTECRAID,
 	)
 
 	DISK_DRIVERS = sets.NewString(

pkg/apis/compute/cloudaccount_const.go

@@ -44,6 +44,7 @@ const (
 	CLOUD_PROVIDER_AZURE     = "Azure"
 	CLOUD_PROVIDER_AWS       = "Aws"
 	CLOUD_PROVIDER_HUAWEI    = "Huawei"
+	CLOUD_PROVIDER_HCSO      = "HCSO"
 	CLOUD_PROVIDER_OPENSTACK = "OpenStack"
 	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
 	CLOUD_PROVIDER_ZSTACK    = "ZStack"
@@ -94,7 +95,7 @@ const (
 var (
 	CLOUD_PROVIDER_VALID_STATUS        = []string{CLOUD_PROVIDER_CONNECTED}
 	CLOUD_PROVIDER_VALID_HEALTH_STATUS = []string{CLOUD_PROVIDER_HEALTH_NORMAL, CLOUD_PROVIDER_HEALTH_NO_PERMISSION}
-	PRIVATE_CLOUD_PROVIDERS            = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA}
+	PRIVATE_CLOUD_PROVIDERS            = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA, CLOUD_PROVIDER_HCSO}
 
 	CLOUD_PROVIDERS = []string{
 		CLOUD_PROVIDER_ONECLOUD,
@@ -105,6 +106,7 @@ var (
 		CLOUD_PROVIDER_AZURE,
 		CLOUD_PROVIDER_AWS,
 		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_HCSO,
 		CLOUD_PROVIDER_OPENSTACK,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_ZSTACK,
@@ -140,6 +142,9 @@ var (
 		CLOUD_PROVIDER_HUAWEI: []string{
 			HOST_TYPE_HUAWEI,
 		},
+		CLOUD_PROVIDER_HCSO: {
+			HOST_TYPE_HCSO,
+		},
 		CLOUD_PROVIDER_OPENSTACK: []string{
 			HOST_TYPE_OPENSTACK,
 		},

pkg/apis/compute/cloudprovider.go

@@ -137,6 +137,8 @@ type CloudproviderDetails struct {
 
 	// 子订阅品牌信息
 	Brand string `json:"brand"`
+
+	ProjectMappingResourceInfo
 }
 
 // 云订阅输入参数

pkg/apis/compute/dnszone.go

@@ -125,7 +125,7 @@ type DnsZoneDetails struct {
 	// 关联vpc数量
 	VpcCount int `json:"vpc_count"`
 	// Cache info
-	CloudCaches []DnsZoneCacheDetails `json:"cloud_caches"`
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type DnsZoneListInput struct {
@@ -141,7 +141,8 @@ type DnsZoneListInput struct {
 	ZoneType string `json:"zone_type"`
 
 	// Filter dns zone By vpc
-	VpcId string `json:"vpc_id"`
+	VpcId     string `json:"vpc_id"`
+	WithCache bool   `json:"with_cache"`
 }
 
 type DnsZoneSyncStatusInput struct {

pkg/apis/compute/filesystem.go

@@ -36,6 +36,9 @@ const (
 	// 删除中
 	NAS_STATUS_DELETING      = "deleting"
 	NAS_STATUS_DELETE_FAILED = "delete_failed"
+
+	NAS_UPDATE_TAGS        = "update_tags"
+	NAS_UPDATE_TAGS_FAILED = "update_tags_fail"
 )
 
 type FileSystemListInput struct {
@@ -104,3 +107,8 @@ type FileSystemDetails struct {
 	Network string
 	Zone    string
 }
+
+type FileSystemRemoteUpdateInput struct {
+	// 是否覆盖替换所有标签
+	ReplaceTags *bool `json:"replace_tags" help:"replace all remote tags"`
+}

pkg/apis/compute/guest_const.go

@@ -159,6 +159,7 @@ const (
 	HYPERVISOR_AZURE     = "azure"
 	HYPERVISOR_AWS       = "aws"
 	HYPERVISOR_HUAWEI    = "huawei"
+	HYPERVISOR_HCSO      = "hcso"
 	HYPERVISOR_OPENSTACK = "openstack"
 	HYPERVISOR_UCLOUD    = "ucloud"
 	HYPERVISOR_ZSTACK    = "zstack"
@@ -175,6 +176,11 @@ const (
 	CPU_MODE_HOST = "host"
 )
 
+const (
+	VM_MACHINE_TYPE_PC  = "pc"
+	VM_MACHINE_TYPE_Q35 = "q35"
+)
+
 var VM_RUNNING_STATUS = []string{VM_START_START, VM_STARTING, VM_RUNNING, VM_BLOCK_STREAM, VM_BLOCK_STREAM_FAIL}
 var VM_CREATING_STATUS = []string{VM_CREATE_NETWORK, VM_CREATE_DISK, VM_START_DEPLOY, VM_DEPLOYING}
 
@@ -189,6 +195,7 @@ var HYPERVISORS = []string{
 	HYPERVISOR_AWS,
 	HYPERVISOR_QCLOUD,
 	HYPERVISOR_HUAWEI,
+	HYPERVISOR_HCSO,
 	HYPERVISOR_OPENSTACK,
 	HYPERVISOR_UCLOUD,
 	HYPERVISOR_ZSTACK,
@@ -219,6 +226,7 @@ var PRIVATE_CLOUD_HYPERVISORS = []string{
 	HYPERVISOR_ZSTACK,
 	HYPERVISOR_OPENSTACK,
 	HYPERVISOR_APSARA,
+	HYPERVISOR_HCSO,
 }
 
 // var HYPERVISORS = []string{HYPERVISOR_ALIYUN}
@@ -234,6 +242,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{
 	HYPERVISOR_AWS:       HOST_TYPE_AWS,
 	HYPERVISOR_QCLOUD:    HOST_TYPE_QCLOUD,
 	HYPERVISOR_HUAWEI:    HOST_TYPE_HUAWEI,
+	HYPERVISOR_HCSO:      HOST_TYPE_HCSO,
 	HYPERVISOR_OPENSTACK: HOST_TYPE_OPENSTACK,
 	HYPERVISOR_UCLOUD:    HOST_TYPE_UCLOUD,
 	HYPERVISOR_ZSTACK:    HOST_TYPE_ZSTACK,
@@ -253,6 +262,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
 	HOST_TYPE_AWS:        HYPERVISOR_AWS,
 	HOST_TYPE_QCLOUD:     HYPERVISOR_QCLOUD,
 	HOST_TYPE_HUAWEI:     HYPERVISOR_HUAWEI,
+	HOST_TYPE_HCSO:       HYPERVISOR_HCSO,
 	HOST_TYPE_OPENSTACK:  HYPERVISOR_OPENSTACK,
 	HOST_TYPE_UCLOUD:     HYPERVISOR_UCLOUD,
 	HOST_TYPE_ZSTACK:     HYPERVISOR_ZSTACK,
@@ -288,3 +298,7 @@ func Hypervisors2HostTypes(hypervisors []string) []string {
 	}
 	return hostTypes
 }
+
+// windows allow a maximal length of 15
+// http://support.microsoft.com/kb/909264
+const MAX_WINDOWS_COMPUTER_NAME_LENGTH = 15

pkg/apis/compute/guest_metadata.go

@@ -4,6 +4,8 @@ const (
 	MIRROR_JOB        = "__mirror_job_status"
 	MIRROR_JOB_READY  = "ready"
 	MIRROR_JOB_FAILED = "failed"
+
+	SSH_PORT = "__ssh_port"
 )
 
 const BASE_INSTANCE_SNAPSHOT_ID = "__base_instance_snapshot_id"

pkg/apis/compute/guest_sshable.go

@@ -44,10 +44,18 @@ type GuestSshableOutput struct {
 	MethodTried []GuestSshableMethodData
 }
 
+type GuestHaveAgentInput struct {
+}
+
+type GuestHaveAgentOutput struct {
+	Have bool
+}
+
 type GuestMakeSshableInput struct {
 	User       string
 	PrivateKey string
 	Password   string
+	Port       int
 }
 
 type GuestMakeSshableOutput struct {
@@ -57,3 +65,11 @@ type GuestMakeSshableOutput struct {
 type GuestMakeSshableCmdOutput struct {
 	ShellCmd string
 }
+
+type GuestSetSshportInput struct {
+	Port int
+}
+
+type GuestSshportOutput struct {
+	Port int
+}

pkg/apis/compute/guests.go

@@ -535,3 +535,10 @@ type ServerResizeDiskInput struct {
 
 	DiskResizeInput
 }
+
+type ServerMigrateNetworkInput struct {
+	// Source network Id
+	Src string `json:"src"`
+	// Destination network Id
+	Dest string `json:"dest"`
+}

pkg/apis/compute/host_const.go

@@ -29,6 +29,7 @@ const (
 	HOST_TYPE_QCLOUD    = "qcloud"
 	HOST_TYPE_AZURE     = "azure"
 	HOST_TYPE_HUAWEI    = "huawei"
+	HOST_TYPE_HCSO      = "hcso"
 	HOST_TYPE_OPENSTACK = "openstack"
 	HOST_TYPE_UCLOUD    = "ucloud"
 	HOST_TYPE_ZSTACK    = "zstack"
@@ -109,6 +110,7 @@ var HOST_TYPES = []string{
 	HOST_TYPE_AWS,
 	HOST_TYPE_QCLOUD,
 	HOST_TYPE_HUAWEI,
+	HOST_TYPE_HCSO,
 	HOST_TYPE_OPENSTACK,
 	HOST_TYPE_UCLOUD,
 	HOST_TYPE_ZSTACK,

pkg/apis/compute/hoststorage.go

@@ -16,6 +16,8 @@ package compute
 
 import (
 	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
 )
 
 type HoststorageDetails struct {
@@ -58,3 +60,10 @@ type HoststorageListInput struct {
 
 	StorageFilterListInput
 }
+
+type HostStorageCreateInput struct {
+	apis.JoinResourceBaseCreateInput
+	StorageId  string `json:"storage_id"`
+	HostId     string `json:"host_id"`
+	MountPoint string `json:"mount_point"`
+}

pkg/apis/compute/network.go

@@ -126,6 +126,8 @@ type NetworkListInput struct {
 
 	// filter by BGP types
 	BgpType []string `json:"bgp_type"`
+
+	HostType string `json:"host_type"`
 }
 
 type NetworkResourceInfoBase struct {
@@ -207,22 +209,11 @@ type NetworkCreateInput struct {
 	BgpType string `json:"bgp_type"`
 }
 
-type NetworkDetails struct {
-	apis.SharableVirtualResourceDetails
-	WireResourceInfo
-
-	SNetwork
-
-	// 是否是内网
-	Exit bool `json:"exit"`
-	// 端口数量
-	Ports int `json:"ports"`
-	// 已使用端口数量
-	PortsUsed int `json:"ports_used"`
-	// 网卡数量
+type SNetworkNics struct {
+	// 虚拟机网卡数量
 	Vnics int `json:"vnics"`
 	// 裸金属网卡数量
-	BmVnics int `json:"bm_nics"`
+	BmVnics int `json:"bm_vnics"`
 	// 负载均衡网卡数量
 	LbVnics int `json:"lb_vnics"`
 	// 浮动Ip网卡数量
@@ -231,6 +222,48 @@ type NetworkDetails struct {
 	// 预留IP数量
 	ReserveVnics int `json:"reserve_vnics"`
 
+	// RDS网卡数量
+	RdsVnics int `json:"rds_vnics"`
+	// NAT网关网卡数量
+	NatVnics      int `json:"nat_vnics"`
+	BmReusedVnics int `json:"bm_reused_vnics"`
+
+	// 弹性网卡数量
+	NetworkinterfaceVnics int `json:"networkinterface_vnics"`
+
+	// 已使用端口数量
+	PortsUsed int `json:"ports_used"`
+
+	Total int `json:"total"`
+}
+
+func (self *SNetworkNics) SumTotal() {
+	self.Total = self.Vnics +
+		self.BmVnics +
+		self.LbVnics +
+		self.LbVnics +
+		self.EipVnics +
+		self.GroupVnics +
+		self.ReserveVnics +
+		self.RdsVnics +
+		self.NetworkinterfaceVnics +
+		self.NatVnics -
+		self.BmReusedVnics
+	self.PortsUsed = self.Total
+}
+
+type NetworkDetails struct {
+	apis.SharableVirtualResourceDetails
+	WireResourceInfo
+
+	SNetwork
+	SNetworkNics
+
+	// 是否是内网
+	Exit bool `json:"exit"`
+	// 端口数量
+	Ports int `json:"ports"`
+
 	// 路由信息
 	Routes    [][]string                 `json:"routes"`
 	Schedtags []SchedtagShortDescDetails `json:"schedtags"`

pkg/apis/compute/network_const.go

@@ -59,6 +59,7 @@ var (
 
 	REGIONAL_NETWORK_PROVIDERS = []string{
 		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_HCSO,
 		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,

pkg/apis/compute/project_mappings.go

@@ -153,6 +153,9 @@ type ProjectMappingDetails struct {
 
 	// 所绑定的云账号列表
 	Accounts []SProjectMappingAccount `json:"accounts"`
+
+	// 所绑定的云订阅列表
+	Managers []SProjectMappingAccount `json:"managers"`
 }
 
 type ProjectMappingCreateInput struct {

pkg/apis/compute/routetable_routesets.go

@@ -70,4 +70,7 @@ type RouteTableRouteSetListInput struct {
 	apis.StatusStandaloneResourceListInput
 	apis.ExternalizedResourceBaseListInput
 	RouteTableFilterList
+	Type        string `json:"type"`
+	NextHopType string `json:"next_hop_type"`
+	NextHopId   string `json:"next_hop_id"`
 }

pkg/apis/compute/secgroup.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"fmt"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
@@ -181,6 +182,7 @@ type SecgroupListInput struct {
 	RegionalFilterListInput
 
 	ManagedResourceListInput
+	WithCache bool `json:"witch_cache"`
 }
 
 type SecurityGroupCacheListInput struct {
@@ -252,6 +254,8 @@ type SecgroupDetails struct {
 	InRules []SecgroupRuleDetails `json:"in_rules"`
 	// 出方向规则信息
 	OutRules []SecgroupRuleDetails `json:"out_rules"`
+
+	CloudCaches []jsonutils.JSONObject `json:"cloud_caches"`
 }
 
 type SecurityGroupResourceInfo struct {

pkg/apis/compute/snapshot.go

@@ -19,13 +19,13 @@ import "yunion.io/x/onecloud/pkg/apis"
 type SnapshotCreateInput struct {
 	apis.VirtualResourceCreateInput
 
-	// 磁盘名称或Id,建议使用Id
+	// 磁盘Id
 	// 目前仅VMware平台不支持创建快照,其余平台磁盘均支持创建快照
 	// required: true
-	Disk string `json:"disk"`
-	// swagger:ignore
 	DiskId string `json:"disk_id"`
 	// swagger:ignore
+	Disk string `json:"disk" yunion-deprecated-by:"disk_id"`
+	// swagger:ignore
 	StorageId string `json:"storage_id"`
 	// swagger:ignore
 	CreatedBy string `json:"created_by"`

pkg/apis/compute/storage_const.go

@@ -68,6 +68,7 @@ const (
 	STORAGE_HUAWEI_SAS   = "SAS"   // 高IO云硬盘
 	STORAGE_HUAWEI_SATA  = "SATA"  // 普通IO云硬盘
 	STORAGE_HUAWEI_GPSSD = "GPSSD" // 通用型SSD
+	STORAGE_HUAWEI_ESSD  = "ESSD"  // 急速型SSD
 
 	// openstack
 	STORAGE_OPENSTACK_ISCSI = "iscsi"

pkg/apis/compute/wire.go

@@ -90,7 +90,8 @@ type WireListInput struct {
 
 	HostResourceInput
 
-	Bandwidth *int `json:"bandwidth"`
+	Bandwidth *int   `json:"bandwidth"`
+	HostType  string `json:"host_type"`
 }
 
 type WireMergeInput struct {

pkg/apis/const.go

@@ -74,10 +74,20 @@ const (
 	OS_ARCH_AARCH64 = "aarch64"
 )
 
-func IsARM(osArch string) bool {
-	return utils.IsInStringArray(osArch, []string{
+var (
+	ARCH_X86 = []string{
+		OS_ARCH_X86,
+		OS_ARCH_I386,
+		OS_ARCH_X86_32,
+		OS_ARCH_X86_64,
+	}
+	ARCH_ARM = []string{
 		OS_ARCH_ARM,
 		OS_ARCH_AARCH32,
 		OS_ARCH_AARCH64,
-	})
+	}
+)
+
+func IsARM(osArch string) bool {
+	return utils.IsInStringArray(osArch, ARCH_ARM)
 }

pkg/apis/devtool/script.go

@@ -29,6 +29,21 @@ type ScriptApplyOutput struct {
 	ScriptApplyId string
 }
 
+type ScriptBatchApplyInput struct {
+	ServerIds []string
+}
+
+type ScriptBatchApplyOutput struct {
+	Results []ScriptBatchApplyResult
+}
+
+type ScriptBatchApplyResult struct {
+	ServerId      string
+	Succeed       bool
+	Reason        string
+	ScriptApplyId string
+}
+
 type ScriptApplyRecoredListInput struct {
 	apis.StatusStandaloneResourceListInput
 	// description: Id of Script
@@ -72,3 +87,12 @@ type SApplyInfo struct {
 	ServerId string
 	TryTimes int
 }
+
+type DevtoolManagerServiceUrlInput struct {
+	ServiceName string
+	ServerId    string
+}
+
+type DevtoolManagerServiceUrlOutput struct {
+	ServiceUrl string
+}