Merge pull request #13169 from swordqiu/hotfix/qj-sync-status-persist-block-stream

fix: guest sync status persistent block_stream status

Makefile

@@ -268,7 +268,8 @@ dep:
 	@$(MAKE) mod
 
 mod:
-	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
+	go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print  | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v sqlchemy))
+	go get -d yunion.io/x/sqlchemy@v1.0.0
 	go mod tidy
 	go mod vendor -v
 

build/docker/Dockerfile.ansibleserver

@@ -1,7 +1,7 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:v1.0.5-2
 
 # install playbook and telegraf install pkg
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.3 /opt/yunion/playbook /opt/yunion/playbook
-COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.3 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/playbook /opt/yunion/playbook
+COPY --from=registry.cn-beijing.aliyuncs.com/yunionio/file-repo:v0.3.4 /opt/yunion/ansible-install-pkg /opt/yunion/ansible-install-pkg
 
 ADD ./_output/alpine-build/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -2,6 +2,8 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
 
 MAINTAINER "Yousong Zhou <zhouyousong@yunionyun.com>"
 
+ENV TZ UTC
+
 # openssh-client, for ansible ssh connection
 # git, ca-certificates, for fetching ansible roles
 RUN set -x \

build/docker/Dockerfile.apigateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/apigateway /opt/yunion/bin/apigateway

build/docker/Dockerfile.baremetal-agent

@@ -1,11 +1,8 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.3
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.3-2
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
 RUN mkdir -p /opt/yunion/bin
 
-#ADD ./_output/bin/baremetal-agent /opt/yunion/bin/baremetal-agent
-#ADD ./_output/bin/.baremetal-agent.bin /opt/yunion/bin/.baremetal-agent.bin
-#ADD ./_output/bin/bundles/baremetal-agent /opt/yunion/bin/bundles/baremetal-agent
-RUN apk add librados librbd
+ENV TZ UTC
 ADD ./_output/alpine-build/bin/baremetal-agent /opt/yunion/bin/baremetal-agent

build/docker/Dockerfile.baremetal-base

@@ -2,18 +2,16 @@ FROM --platform=linux/amd64 registry.cn-beijing.aliyuncs.com/yunionio/centos-bui
 RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 #RUN yum install -y http://192.168.23.50:8083/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3-3.13.5
 
 MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
 
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk update && \
-    apk add --no-cache ipmitool ethtool tzdata ca-certificates cdrkit coreutils && \
+    apk add --no-cache ipmitool ethtool tzdata ca-certificates cdrkit coreutils librados librbd && \
     rm -rf /var/cache/apk/*
 
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
 RUN mkdir -p /opt/cloud/yunion/baremetal
 
 # udpate latest pci.ids

build/docker/Dockerfile.cloudevent

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/cloudevent /opt/yunion/bin/cloudevent

build/docker/Dockerfile.cloudid

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 COPY ./build/cloudid/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/cloudid /opt/yunion/bin/cloudid

build/docker/Dockerfile.cloudmon

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.cloudnet

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/cloudnet /opt/yunion/bin/cloudnet

build/docker/Dockerfile.cloudproxy

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/cloudproxy /opt/yunion/bin/cloudproxy

build/docker/Dockerfile.devtool

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/devtool /opt/yunion/bin/devtool

build/docker/Dockerfile.esxi-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 COPY ./build/esxi-agent/root/opt/ /opt/
 

build/docker/Dockerfile.glance

@@ -3,5 +3,6 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/torrent:20210815.0
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
 RUN apk add --no-cache s3fs-fuse --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing
+ENV TZ UTC
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/glance /opt/yunion/bin/glance

build/docker/Dockerfile.host

@@ -2,7 +2,7 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/host-base:v0.4.0
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 
-ENV TZ Asia/Shanghai
+ENV TZ UTC
 
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/host /opt/yunion/bin/host

build/docker/Dockerfile.host-deployer

@@ -2,7 +2,7 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:1.1
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 
-ENV TZ Asia/Shanghai
+ENV TZ UTC
 
 RUN mkdir -p /opt/yunion/bin
 

build/docker/Dockerfile.host-image

@@ -0,0 +1,9 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
+
+RUN mkdir -p /opt/yunion/bin
+
+ENV TZ UTC
+
+ADD ./_output/bin/host-image /opt/yunion/bin/host-image
+ADD ./_output/bin/.host-image.bin /opt/yunion/bin/.host-image.bin
+ADD ./_output/bin/bundles/host-image /opt/yunion/bin/bundles/host-image

build/docker/Dockerfile.keystone

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/keystone /opt/yunion/bin/keystone

build/docker/Dockerfile.logger

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/logger /opt/yunion/bin/logger

build/docker/Dockerfile.monitor

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 COPY ./build/monitor/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/monitor /opt/yunion/bin/monitor

build/docker/Dockerfile.notify

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 COPY ./build/notify/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.onecloud-base

@@ -3,12 +3,12 @@ FROM alpine:3.13.5
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
-ENV TZ Asia/Shanghai
+ENV TZ UTC
 
 RUN mkdir -p /opt/yunion/bin
 
+RUN sed -i 's!https://dl-cdn.alpinelinux.org/!https://mirrors.ustc.edu.cn/!g' /etc/apk/repositories
+
 RUN apk update && \
     apk add --no-cache tzdata curl busybox-extras tcpdump strace ca-certificates && \
     rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.region

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 COPY ./build/region/root/opt/ /opt/
 ADD ./_output/alpine-build/bin/region /opt/yunion/bin/region

build/docker/Dockerfile.region-dns

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/region-dns /opt/yunion/bin/region-dns

build/docker/Dockerfile.s3gateway

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/s3gateway /opt/yunion/bin/s3gateway

build/docker/Dockerfile.scheduler

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/scheduler /opt/yunion/bin/scheduler

build/docker/Dockerfile.torrent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 MAINTAINER "Zexi Li <zexi.li@icloud.com>"
 

build/docker/Dockerfile.vpcagent

@@ -2,5 +2,6 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/openvswitch:2.10.5-1
 
 MAINTAINER "Yousong Zhou <zhouyousong@yunion.cn>"
 
+ENV TZ UTC
 RUN mkdir -p /opt/yunion/bin
 ADD ./_output/alpine-build/bin/vpcagent /opt/yunion/bin/vpcagent

build/docker/Dockerfile.webconsole

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/webconsole-base:v0.2
+FROM registry.cn-beijing.aliyuncs.com/yunionio/webconsole-base:v0.2-1
 
-ADD ./_output/alpine-build/bin/webconsole /opt/yunion/bin/webconsole
+ADD ./_output/alpine-build/bin/webconsole /opt/yunion/bin/webconsole

build/docker/Dockerfile.webconsole-base

@@ -2,11 +2,9 @@ FROM registry.cn-beijing.aliyuncs.com/yunionio/kubectl:1.18.6
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 
-ENV TZ Asia/Shanghai
+ENV TZ UTC
 
 RUN mkdir -p /opt/yunion/bin
 
 RUN apk add --no-cache sshpass ipmitool openssh-client tzdata ca-certificates && \
     rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.yunionconf

@@ -1,3 +1,3 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.3.5-1
 
 ADD ./_output/alpine-build/bin/yunionconf /opt/yunion/bin/yunionconf

build/docker/Makefile

@@ -7,12 +7,12 @@ debian10-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/debian10-base:1.0 -f ./Dockerfile.debian-base .
 
-ONECLOUD_BASE_VERSION = v0.3.5
+ONECLOUD_BASE_VERSION = v0.3-3.13.5
 
 onecloud-base:
 	$(DOCKER_BUILDX)/onecloud-base:$(ONECLOUD_BASE_VERSION) -f ./Dockerfile.onecloud-base .
 
-ANSIBLESERVER_BASE = v1.0.5
+ANSIBLESERVER_BASE = v1.0.5-2
 
 ansibleserver-base:
 	$(DOCKER_BUILDX)/ansibleserver-base:$(ANSIBLESERVER_BASE) -f ./Dockerfile.ansibleserver-base .
@@ -23,7 +23,11 @@ climc-base:
 	docker buildx build --platform linux/arm64,linux/amd64 --push \
 		-t registry.cn-beijing.aliyuncs.com/yunionio/climc-base:$(CLIMC_BASE_VERSION) -f ./Dockerfile.climc-base .
 
-BAREMETAL_BASE_VERSION = v0.3.3
+WEBCONSOLE_BASE_VERSION = v0.2-1
+webconsole-base:
+	$(DOCKER_BUILDX)/webconsole-base:$(WEBCONSOLE_BASE_VERSION) -f ./Dockerfile.webconsole-base .
+
+BAREMETAL_BASE_VERSION = v0.3.3-2
 
 baremetal-base:
 	$(DOCKER_BUILDX)/baremetal-base:$(BAREMETAL_BASE_VERSION) -f ./Dockerfile.baremetal-base .

build/notify/root/opt/yunion/share/local-templates/content@cn/COMMON.tmpl

@@ -3,7 +3,7 @@
 {{- if $d.brand -}}
 {{ $d.brand }}平台
 {{- end -}}
-{{ .resource_type_display }}{{ $d.name }}已经{{ .action_display }}{{.result_display}}
+{{ .resource_type_display }}{{ $d.name }}{{ .action_display }}{{.result_display}}
 {{- if eq .result "failed" -}}
 ，请尽快前往控制台进行处理
 {{- end -}}

build/notify/root/opt/yunion/share/local-templates/content@cn/CREATE.tmpl

@@ -4,7 +4,7 @@
 {{- if $d.brand -}}
 {{ $d.brand }}平台
 {{- end -}}
-{{ .resource_type_display }}{{ $d.name }}已经创建成功
+{{ .resource_type_display }}{{ $d.name }}创建成功
 {{- if $d.private_dns -}}
 ，内网地址为{{ $d.private_dns }}:{{ $d.private_connect_port }}
 {{- end -}}

build/notify/root/opt/yunion/share/local-templates/content@cn/DELETE_WITH_IP.tmpl

@@ -19,4 +19,4 @@
 {{- if $d.ip_addr -}}
 ，IP地址为{{ $d.ip_addr }}
 {{- end -}}
-{{ .resource_type_display }}{{ $d.name }}已经删除成功。
+{{ .resource_type_display }}{{ $d.name }}删除成功

build/notify/root/opt/yunion/share/local-templates/content@cn/PENDING_DELETE.tmpl

@@ -0,0 +1,14 @@
+{{- $d := .resource_details -}}
+您在{{ $d.project }}项目的
+
+{{- if $d.ips -}}
+IP地址为{{ $d.ips }}的
+{{- end -}}
+
+{{- if $d.ip_addr -}}
+IP地址为{{ $d.ip_addr }}的
+{{- end -}}
+{{- if $d.brand -}}
+{{ $d.brand }}平台
+{{- end -}}
+{{ .resource_type_display }}{{ $d.name }}成功加入回收站

build/notify/root/opt/yunion/share/local-templates/content@cn/UPDATE.tmpl

@@ -3,7 +3,7 @@
 {{- if $d.brand -}}
 {{ $d.brand }}平台
 {{- end -}}
-{{ .resource_type_display }}{{ $d.name }}已经{{ .action_display }}成功
+{{ .resource_type_display }}{{ $d.name }}{{ .action_display }}成功
 {{- if $d.account -}}
 ，帐号为{{ $d.account }}
 {{- end -}}

build/notify/root/opt/yunion/share/local-templates/content@en/PENDING_DELETE.tmpl

@@ -0,0 +1,22 @@
+{{- $d := .resource_details -}}
+Your {{ if $d.brand -}} {{ $d.brand }} {{ end -}} {{ .resource_type_display }} {{ $d.name }} 
+{{ if $d.project -}} in project {{ $d.project }} {{ end -}} has been added to recycle bin successfully
+{{- if $d.private_dns -}}
+, the intranet address is {{ $d.private_dns }}:{{ $d.private_connect_port }}
+{{- end -}}
+
+{{- if $d.public_dns -}}
+, the external address is {{ $d.public_dns }}:{{ $d.public_connect_port }}
+{{- end -}}
+
+{{- if and $d.address_type $d.address -}}
+, the service address is {{ $d.address_type }}{{ $d.address }}
+{{- end -}}
+
+{{- if $d.ips -}}
+, the IP address is {{ $d.ips }}
+{{- end -}}
+
+{{- if $d.ip_addr -}}
+, the IP address is {{ $d.ip_addr }}
+{{- end -}}

build/notify/root/opt/yunion/share/local-templates/title@cn/PENDING_DELETE.tmpl

@@ -0,0 +1,5 @@
+{{- $d := .resource_details -}}
+{{- if $d.project -}}
+{{ $d.project }}项目的
+{{- end -}}
+{{ .resource_type_display }}{{ $d.name }}成功加入回收站

build/notify/root/opt/yunion/share/local-templates/title@en/PENDING_DELETE.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+The {{ .resource_type_display }} {{ .Name }} {{ if $d.project }} in poject {{ $d.project }} {{ end -}} has been added to recycle bin successfully

cmd/aliyuncli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/aliyun"
 	_ "yunion.io/x/onecloud/pkg/multicloud/aliyun/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -85,12 +90,27 @@ func newClient(options *BaseOptions) (*aliyun.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := aliyun.NewAliyunClient(
 		aliyun.NewAliyunClientConfig(
 			options.CloudEnv,
 			options.AccessKey,
 			options.Secret,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/apsaracli/main.go

@@ -16,8 +16,12 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -87,13 +91,28 @@ func newClient(options *BaseOptions) (*apsara.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := apsara.NewApsaraClient(
 		apsara.NewApsaraClientConfig(
 			options.AccessKey,
 			options.Secret,
 			options.Endpoint,
 			options.SApsaraEndpoints,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/awscli/main.go

@@ -16,11 +16,16 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/aws"
 	_ "yunion.io/x/onecloud/pkg/multicloud/aws/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -87,13 +92,28 @@ func newClient(options *BaseOptions) (*aws.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := aws.NewAwsClient(
 		aws.NewAwsClientConfig(
 			options.AccessUrl,
 			options.AccessKey,
 			options.Secret,
 			options.AccountId,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/azurecli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/azure"
 	_ "yunion.io/x/onecloud/pkg/multicloud/azure/shell"
 	"yunion.io/x/onecloud/pkg/util/printutils"
@@ -100,6 +105,16 @@ func newClient(options *BaseOptions) (*azure.SRegion, error) {
 		return nil, fmt.Errorf("Missing Cloud Environment")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := azure.NewAzureClient(
 		azure.NewAzureClientConfig(
 			options.CloudEnv,
@@ -108,7 +123,12 @@ func newClient(options *BaseOptions) (*azure.SRegion, error) {
 			options.ApplicationKey,
 		).
 			SubscriptionId(options.SubscriptionID).
-			Debug(options.Debug),
+			Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 
 	if err != nil {

cmd/climc/shell/cloudproxy/proxy_agents.go

@@ -28,4 +28,5 @@ func init() {
 	cmd.List(&options.ProxyAgentListOptions{})
 	cmd.Show(&options.ProxyAgentShowOptions{})
 	cmd.Delete(&options.ProxyAgentDeleteOptions{})
+	cmd.Perform("purge-forwards", &options.ProxyAgentPurgeForwardsOptions{})
 }

cmd/climc/shell/compute/cdn_domains.go

@@ -17,10 +17,15 @@ package compute
 import (
 	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
 	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
 	cmd := shell.NewResourceCmd(&modules.CDNDomains)
 	cmd.List(&compute.CDNDomainListOptions{})
+	cmd.Show(&options.BaseIdOptions{})
+	cmd.Update(&compute.CDNDomainUpdateOptions{})
+	cmd.Delete(&options.BaseIdOptions{})
+	cmd.Create(&compute.CDNDomainCreateOptions{})
 }

cmd/climc/shell/compute/common.go

@@ -21,8 +21,8 @@ import (
 
 var (
 	R                 = shell.R
-	printList         = printutils.PrintJSONList
-	printObject       = printutils.PrintJSONObject
+	printList         = shell.PrintList
+	printObject       = shell.PrintObject
 	printBatchResults = printutils.PrintJSONBatchResults
 
 	InvalidUpdateError   = shell.InvalidUpdateError

cmd/climc/shell/compute/elastic_search.go

@@ -27,4 +27,5 @@ func init() {
 	cmd.Show(&compute.ElasticSearchIdOption{})
 	cmd.Delete(&compute.ElasticSearchIdOption{})
 	cmd.Perform("syncstatus", &compute.ElasticSearchIdOption{})
+	cmd.Get("access-info", &compute.ElasticSearchIdOption{})
 }

cmd/climc/shell/compute/globalvpcs.go

@@ -15,95 +15,16 @@
 package compute
 
 import (
-	"fmt"
-
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/onecloud/pkg/mcclient/options"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
-	type GlobalVpcListOptions struct {
-		options.BaseListOptions
-	}
-	R(&GlobalVpcListOptions{}, "global-vpc-list", "List global vpcs", func(s *mcclient.ClientSession, args *GlobalVpcListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.GlobalVpcs.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.GlobalVpcs.GetColumns(s))
-		return nil
-	})
-
-	type GlobalVpcShowOptions struct {
-		ID string `help:"ID or Name of globalvpc"`
-	}
-	R(&GlobalVpcShowOptions{}, "global-vpc-show", "Show details of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
-		result, err := modules.GlobalVpcs.GetById(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type GlobalVpcPublicOptions struct {
-		ID            string   `help:"ID or name of global vpc" json:"-"`
-		Scope         string   `help:"sharing scope" choices:"system|domain"`
-		SharedDomains []string `help:"share to domains"`
-	}
-	R(&GlobalVpcPublicOptions{}, "global-vpc-public", "Make global vpc public", func(s *mcclient.ClientSession, args *GlobalVpcPublicOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.GlobalVpcs.PerformAction(s, args.ID, "public", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type GlobalVpcPrivateOptions struct {
-		ID string `help:"ID or name of global vpc" json:"-"`
-	}
-	R(&GlobalVpcPrivateOptions{}, "global-vpc-private", "Make global vpc private", func(s *mcclient.ClientSession, args *GlobalVpcPrivateOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.GlobalVpcs.PerformAction(s, args.ID, "private", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&GlobalVpcShowOptions{}, "global-vpc-change-owner-candidate-domains", "Show candiate domains of a global vpc for changing owner", func(s *mcclient.ClientSession, args *GlobalVpcShowOptions) error {
-		result, err := modules.GlobalVpcs.GetSpecific(s, args.ID, "change-owner-candidate-domains", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type GlobalVpcChangeOwnerOptions struct {
-		ID            string `help:"ID or name of vpc" json:"-"`
-		ProjectDomain string `json:"project_domain" help:"target domain"`
-	}
-	R(&GlobalVpcChangeOwnerOptions{}, "global-vpc-change-owner", "Change owner domain of a global vpc", func(s *mcclient.ClientSession, args *GlobalVpcChangeOwnerOptions) error {
-		if len(args.ProjectDomain) == 0 {
-			return fmt.Errorf("empty project_domain")
-		}
-		params := jsonutils.Marshal(args)
-		ret, err := modules.GlobalVpcs.PerformAction(s, args.ID, "change-owner", params)
-		if err != nil {
-			return err
-		}
-		printObject(ret)
-		return nil
-	})
+	cmd := shell.NewResourceCmd(&modules.GlobalVpcs).WithKeyword("global-vpc")
+	cmd.List(&options.GlobalVpcListOptions{})
+	cmd.Show(&options.GlobalVpcIdOption{})
+	cmd.Create(&options.GlobalVpcCreateOptions{})
+	cmd.Perform("syncstatus", &options.GlobalVpcIdOption{})
+	cmd.Get("change-owner-candidate-domains", &options.GlobalVpcIdOption{})
 }

cmd/climc/shell/compute/hosts.go

@@ -48,6 +48,7 @@ func init() {
 	cmd.Perform("reset", &options.BaseIdOptions{})
 	cmd.BatchDelete(&options.BaseIdsOptions{})
 	cmd.Perform("remove-all-netifs", &options.BaseIdOptions{})
+	cmd.Perform("probe-isolated-devices", &options.BaseIdOptions{})
 
 	cmd.BatchPerform("enable", &options.BaseIdsOptions{})
 	cmd.BatchPerform("disable", &options.BaseIdsOptions{})

cmd/climc/shell/compute/instance_group.go

@@ -15,113 +15,28 @@
 package compute
 
 import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
-	type InstanceGroupListOptions struct {
-		options.BaseListOptions
+	cmd := shell.NewResourceCmd(&modules.InstanceGroups)
+	cmd.List(&compute.InstanceGroupListOptions{})
+	cmd.Show(&options.BaseShowOptions{})
+	cmd.GetMetadata(&options.BaseIdOptions{})
+	cmd.Create(&compute.InstanceGroupCreateOptions{})
+	cmd.Update(&compute.InstanceGroupUpdateOptions{})
+	cmd.Delete(&options.BaseIdOptions{})
 
-		ServiceType string `help:"Service Type"`
-		ParentId    string `help:"Parent ID"`
-		ZoneId      string `help:"Zone ID"`
-		Server      string `help:"Guest ID or Name"`
-	}
+	cmd.Perform("bind-guests", &compute.InstanceGroupBindGuestsOptions{})
+	cmd.Perform("unbind-guests", &compute.InstanceGroupBindGuestsOptions{})
 
-	R(&InstanceGroupListOptions{}, "instance-group-list", "List instance group", func(s *mcclient.ClientSession,
-		args *InstanceGroupListOptions) error {
-
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.InstanceGroup.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.InstanceGroup.GetColumns(s))
-		return nil
-	})
-
-	type InstanceGroupShowOptions struct {
-		ID string `help:"ID or Name of instance group"`
-	}
-	R(&InstanceGroupShowOptions{}, "instance-group-show", "Show details of a instance group",
-		func(s *mcclient.ClientSession, args *InstanceGroupShowOptions) error {
-
-			result, err := modules.InstanceGroup.GetById(s, args.ID, nil)
-			if err != nil {
-				return err
-			}
-			printObject(result)
-			return nil
-		})
-
-	type InstanceGroupCreateOptions struct {
-		NAME string `help:"name of instance group"`
-
-		ZoneId          string `help:"zone id" json:"zone_id"`
-		ServiceType     string `help:"service type"`
-		ParentId        string `help:"parent id"`
-		SchedStrategy   string `help:"scheduler strategy"`
-		Granularity     string `help:"the upper limit number of guests with this group in a host"`
-		ForceDispersion bool   `help:"force to make guest dispersion"`
-	}
-
-	R(&InstanceGroupCreateOptions{}, "instance-group-create", "Create a instance group",
-		func(s *mcclient.ClientSession, args *InstanceGroupCreateOptions) error {
-			params, err := options.StructToParams(args)
-			if err != nil {
-				return err
-			}
-			result, err := modules.InstanceGroup.Create(s, params)
-			if err != nil {
-				return err
-			}
-			printObject(result)
-			return nil
-		},
-	)
-
-	R(&InstanceGroupShowOptions{}, "instance-group-delete", "delete a instance group",
-		func(s *mcclient.ClientSession, args *InstanceGroupShowOptions) error {
-			result, err := modules.InstanceGroup.Delete(s, args.ID, nil)
-			if err != nil {
-				return err
-			}
-			printObject(result)
-			return nil
-		},
-	)
-
-	type InstanceGroupUpdateOptions struct {
-		ID              string `help:"ID or Name of servers to update" json:"-"`
-		Name            string `help:"New name to change"`
-		Granularity     string `help:"the upper limit number of guests with this group in a host"`
-		ForceDispersion string `help:"force to make guest dispersion" choices:"yes|no" json:"-"`
-	}
-
-	R(&InstanceGroupUpdateOptions{}, "instance-group-update", "update a instance group",
-		func(s *mcclient.ClientSession, args *InstanceGroupUpdateOptions) error {
-			params, err := options.StructToParams(args)
-			if err != nil {
-				return err
-			}
-			if args.ForceDispersion == "yes" {
-				params.Set("force_dispersion", jsonutils.JSONTrue)
-			} else {
-				params.Set("force_dispersion", jsonutils.JSONFalse)
-			}
-			ret, err := modules.InstanceGroup.Update(s, args.ID, params)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
+	cmd.Perform("attachnetwork", &compute.InstanceGroupAttachnetworkOptions{})
+	cmd.Perform("detachnetwork", &compute.InstanceGroupDetachnetworkOptions{})
 
+	cmd.Perform("create-eip", &compute.InstanceGroupCreateEipOptions{})
+	cmd.Perform("associate-eip", &compute.InstanceGroupAssociateEipOptions{})
+	cmd.Perform("dissociate-eip", &compute.InstanceGroupDissociateEipOptions{})
 }

cmd/climc/shell/compute/instance_group_guest.go

@@ -12,14 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package shell
+package compute
 
-import "yunion.io/x/onecloud/pkg/util/printutils"
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
+)
 
-func printList(data interface{}, total, offset, limit int, columns []string) {
-	printutils.PrintInterfaceList(data, total, offset, limit, columns)
-}
-
-func printObject(obj interface{}) {
-	printutils.PrintInterfaceObject(obj)
+func init() {
+	cmd := shell.NewJointCmd(&modules.InstanceGroupGuests)
+	cmd.List(&compute.InstanceGroupGuestListOptions{})
 }

cmd/climc/shell/compute/instance_group_network.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
+)
+
+func init() {
+	cmd := shell.NewJointCmd(&modules.InstancegroupNetworks)
+	cmd.List(&compute.InstanceGroupNetworkListOptions{})
+}

cmd/climc/shell/compute/isolatedevices.go

@@ -30,6 +30,7 @@ func init() {
 		Host   string `help:"Host ID or Name"`
 		Region string `help:"Cloudregion ID or Name"`
 		Zone   string `help:"Zone ID or Name"`
+		Server string `help:"Server ID or Name"`
 	}
 	R(&DeviceListOptions{}, "isolated-device-list", "List isolated devices like GPU", func(s *mcclient.ClientSession, args *DeviceListOptions) error {
 		var params *jsonutils.JSONDict
@@ -56,6 +57,9 @@ func init() {
 		if args.Zone != "" {
 			params.Add(jsonutils.NewString(args.Zone), "zone")
 		}
+		if args.Server != "" {
+			params.Add(jsonutils.NewString(args.Server), "guest_id")
+		}
 		result, err := modules.IsolatedDevices.List(s, params)
 		if err != nil {
 			return err
@@ -76,12 +80,12 @@ func init() {
 		return nil
 	})
 
-	R(&DeviceShowOptions{}, "isolated-device-delete", "Delete a isolated device", func(s *mcclient.ClientSession, args *DeviceShowOptions) error {
-		result, err := modules.IsolatedDevices.Delete(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
+	type DeviceDeleteOptions struct {
+		IDS []string `help:"IDs of the isolated device"`
+	}
+	R(&DeviceDeleteOptions{}, "isolated-device-delete", "Delete a isolated device", func(s *mcclient.ClientSession, args *DeviceDeleteOptions) error {
+		result := modules.IsolatedDevices.BatchDelete(s, args.IDS, nil)
+		printBatchResults(result, modules.IsolatedDevices.GetColumns(s))
 		return nil
 	})
 

cmd/climc/shell/compute/server_forward.go

@@ -47,7 +47,7 @@ func dump(input jsonutils.JSONObject) (*forwardInfo, error) {
 	}
 
 	if ret.ProxyPort <= 0 {
-		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyAddr)
+		return nil, errors.Errorf("invalid proxy_port %d", ret.ProxyPort)
 	}
 
 	return ret, nil

cmd/climc/shell/compute/serverisolatedevices.go

@@ -48,7 +48,7 @@ func init() {
 	R(&ServerDetachDeviceOptions{}, "server-detach-isolated-device", "Detach a isolated device from a virtual server", func(s *mcclient.ClientSession, args *ServerDetachDeviceOptions) error {
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewString(args.DEVICE), "device")
-		srv, err := modules.Servers.PerformAction(s, args.SERVER, "detach-isolated_device", params)
+		srv, err := modules.Servers.PerformAction(s, args.SERVER, "detach-isolated-device", params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/compute/servers.go

@@ -90,6 +90,9 @@ func init() {
 	cmd.Perform("make-sshable", &options.ServerMakeSshableOptions{})
 	cmd.Perform("migrate-network", &options.ServerMigrateNetworkOptions{})
 	cmd.Perform("set-sshport", &options.ServerSetSshportOptions{})
+	cmd.Perform("have-agent", &options.ServerHaveAgentOptions{})
+	cmd.Perform("change-disk-storage", &options.ServerChangeDiskStorageOptions{})
+	cmd.Perform("probe-isolated-devices", &options.ServerIdOptions{})
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))

cmd/climc/shell/compute/vpcs.go

@@ -36,6 +36,7 @@ func init() {
 	cmd.Perform("public", &options.BasePublicOptions{})
 	cmd.Perform("change-owner", &options.VpcChangeOwnerOptions{})
 	cmd.Get("vpc-change-owner-candidate-domains", &options.VpcIdOptions{})
+	cmd.Get("topology", &options.VpcIdOptions{})
 
 	R(&options.ResourceMetadataOptions{}, "vpc-set-user-metadata", "Set metadata of a vpc", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error {
 		params, err := opts.Params()

cmd/climc/shell/compute/wires.go

@@ -32,4 +32,5 @@ func init() {
 	cmd.Perform("change-owner-candidate-domains", new(options.WireOptions))
 	cmd.PerformWithKeyword("merge", "merge-from", new(options.WireMergeOptions))
 	cmd.Perform("merge-network", new(options.WireOptions))
+	cmd.Get("topology", &options.WireOptions{})
 }

cmd/climc/shell/helper.go

@@ -160,7 +160,7 @@ func (cmd ResourceCmd) List(args IListOpt) {
 			ExportList(result, exportFile, args.GetExportKeys(), args.GetExportTexts(), man.GetColumns(s))
 			return nil
 		}
-		printList(result, man.GetColumns(s))
+		PrintList(result, man.GetColumns(s))
 		return nil
 	}
 	cmd.Run("list", args, callback)
@@ -184,7 +184,7 @@ func (cmd ResourceCmd) create(s *mcclient.ClientSession, args ICreateOpt) error
 	if err != nil {
 		return err
 	}
-	printObject(ret)
+	PrintObject(ret)
 	return nil
 }
 
@@ -240,7 +240,7 @@ func (cmd ResourceCmd) GetProperty(args IPropertyOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.RunWithDesc(args.Property(), fmt.Sprintf("Get property of a %s", man.GetKeyword()), args, callback)
@@ -257,7 +257,7 @@ func (cmd ResourceCmd) Show(args IShowOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.RunWithDesc("show", fmt.Sprintf("Show details of a %s", man.GetKeyword()), args, callback)
@@ -274,7 +274,7 @@ func (cmd ResourceCmd) ClassShow(args IShowOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.RunWithDesc(args.GetId(), fmt.Sprintf("Show %s of a %s", args.GetId(), man.GetKeyword()), args, callback)
@@ -320,7 +320,7 @@ func (cmd ResourceCmd) Custom(action TCustomAction, funcname string, args IGetAc
 		if reterr.IsNil() {
 			v, ok := retobj.Interface().(jsonutils.JSONObject)
 			if ok {
-				printObject(v)
+				PrintObject(v)
 				return nil
 			}
 		}
@@ -349,7 +349,7 @@ func (cmd ResourceCmd) Delete(args IDeleteOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.RunWithDesc("delete", fmt.Sprintf("Delete %s", man.GetKeyword()), args, callback)
@@ -461,7 +461,7 @@ func (cmd ResourceCmd) GetWithCustomShow(specific string, show func(data jsonuti
 }
 
 func (cmd ResourceCmd) Get(specific string, args IGetOpt) {
-	cmd.GetWithCustomShow(specific, printObject, args)
+	cmd.GetWithCustomShow(specific, PrintObject, args)
 }
 
 type IUpdateOpt interface {
@@ -485,7 +485,7 @@ func (cmd ResourceCmd) UpdateWithKeyword(keyword string, args IUpdateOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.Run(keyword, args, callback)
@@ -511,7 +511,7 @@ func (cmd ResourceCmd) GetMetadata(args IMetadataOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(ret)
+		PrintObject(ret)
 		return nil
 	}
 	cmd.RunWithDesc("metadata", fmt.Sprintf("Show metadata of a %s", man.GetKeyword()), args, callback)
@@ -612,7 +612,7 @@ func (cmd JointCmd) List(args IJointListOpt) {
 		if err != nil {
 			return err
 		}
-		printList(result, man.GetColumns(s))
+		PrintList(result, man.GetColumns(s))
 		return nil
 	}
 	cmd.RunWithDesc("list", fmt.Sprintf("list %s %s pairs", man.MasterManager().KeyString(), man.SlaveManager().KeyString()), args, callback)
@@ -635,7 +635,7 @@ func (cmd JointCmd) Show(args IJointShowOpt) {
 		if err != nil {
 			return err
 		}
-		printObject(result)
+		PrintObject(result)
 		return nil
 	}
 	cmd.Run("show", args, callback)

cmd/climc/shell/identity/groups.go

@@ -193,39 +193,6 @@ func init() {
 		return nil
 	})
 
-	type GroupBindGuestsOptions struct {
-		ID    string   `help:"ID"`
-		Guest []string `help:"ID or Name of Guest"`
-	}
-
-	R(&GroupBindGuestsOptions{}, "instance-group-bind-guests", "bind instancegroup to guests",
-		func(s *mcclient.ClientSession, opts *GroupBindGuestsOptions) error {
-			params, err := options.StructToParams(opts)
-			if err != nil {
-				return err
-			}
-			sp, err := modules.InstanceGroup.PerformAction(s, opts.ID, "bind-guests", params)
-			if err != nil {
-				return err
-			}
-			printObject(sp)
-			return nil
-		})
-
-	R(&GroupBindGuestsOptions{}, "instance-group-unbind-guests", "bind instancegroup to guests",
-		func(s *mcclient.ClientSession, opts *GroupBindGuestsOptions) error {
-			params, err := options.StructToParams(opts)
-			if err != nil {
-				return err
-			}
-			sp, err := modules.InstanceGroup.PerformAction(s, opts.ID, "unbind-guests", params)
-			if err != nil {
-				return err
-			}
-			printObject(sp)
-			return nil
-		})
-
 	type GroupAddUsersOptions struct {
 		GROUP string   `help:"group Id or Name" json:"-"`
 		USER  []string `help:"list of user id or name" json:"user"`

cmd/climc/shell/image/images.go

@@ -477,4 +477,16 @@ func init() {
 		printObject(srv)
 		return nil
 	})
+
+	type ImageProbeOptions struct {
+		ID string `help:"ID or name of image to probe"`
+	}
+	R(&ImageProbeOptions{}, "image-probe", "Start image probe task", func(s *mcclient.ClientSession, opts *ImageProbeOptions) error {
+		img, err := modules.Images.PerformAction(s, opts.ID, "probe", nil)
+		if err != nil {
+			return err
+		}
+		printObject(img)
+		return nil
+	})
 }

cmd/climc/shell/utils.go

@@ -41,18 +41,26 @@ func OutputFormat(s string) {
 	outputFormat = s
 }
 
-func printList(list *modulebase.ListResult, columns []string) {
-	printutils.PrintJSONList(list, columns)
+func PrintList(list *modulebase.ListResult, columns []string) {
+	switch outputFormat {
+	case OUTPUT_FORMAT_TABLE:
+		printutils.PrintJSONList(list, columns)
+	case OUTPUT_FORMAT_JSON:
+		fmt.Print(jsonutils.Marshal(list).PrettyString())
+		fmt.Print("\n")
+	default:
+		fmt.Fprintf(os.Stderr, "unknown output format: %q\n", outputFormat)
+	}
 }
 
-func printObject(obj jsonutils.JSONObject) {
+func PrintObject(obj jsonutils.JSONObject) {
 	switch outputFormat {
 	case OUTPUT_FORMAT_TABLE:
 		printutils.PrintJSONObject(obj)
 	case OUTPUT_FORMAT_KV:
 		printObjectFmtKv(obj)
 	case OUTPUT_FORMAT_JSON:
-		fmt.Print(obj.String())
+		fmt.Print(obj.PrettyString())
 		fmt.Print("\n")
 	case OUTPUT_FORMAT_FLATTEN_TABLE:
 		printObjectRecursive(obj)
@@ -88,7 +96,7 @@ func printObjectFmtKv(obj jsonutils.JSONObject) {
 }
 
 func printObjectRecursive(obj jsonutils.JSONObject) {
-	printutils.PrintJSONObjectRecursive(obj)
+	PrintObject(obj)
 }
 
 func printObjectRecursiveEx(obj jsonutils.JSONObject, cb printutils.PrintJSONObjectRecursiveExFunc) {

cmd/cloudpodscli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/cloudpods"
 	_ "yunion.io/x/onecloud/pkg/multicloud/cloudpods/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -88,12 +93,27 @@ func newClient(options *BaseOptions) (*cloudpods.SRegion, error) {
 		return nil, fmt.Errorf("Missing access secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := cloudpods.NewCloudpodsClient(
 		cloudpods.NewCloudpodsClientConfig(
 			options.AuthURL,
 			options.AccessKey,
 			options.AccessSecret,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/ctyuncli/main.go

@@ -16,16 +16,23 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/ctyun"
 	_ "yunion.io/x/onecloud/pkg/multicloud/ctyun/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 type BaseOptions struct {
+	cloudprovider.SCtyunExtraOptions
+
 	Help       bool   `help:"Show help" default:"false"`
 	Debug      bool   `help:"Show debug" default:"false"`
 	AccessKey  string `help:"Access key" default:"$CTYUN_ACCESS_KEY"`
@@ -84,10 +91,25 @@ func newClient(options *BaseOptions) (*ctyun.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := ctyun.NewSCtyunClient(
 		ctyun.NewSCtyunClientConfig(
-			options.AccessKey, options.Secret,
-		).Debug(options.Debug),
+			options.AccessKey, options.Secret, &options.SCtyunExtraOptions,
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/ecloudcli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/ecloud"
 	_ "yunion.io/x/onecloud/pkg/multicloud/ecloud/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -84,10 +89,25 @@ func newClient(options *Options) (*ecloud.SRegion, error) {
 		return nil, fmt.Errorf("Missing access secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := ecloud.NewEcloudClient(
 		ecloud.NewEcloudClientConfig(
 			ecloud.NewRamRoleSigner(options.AccessKey, options.AccessSecret),
-		).SetDebug(options.Debug),
+		).SetDebug(options.Debug).
+			SetCloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/esxicli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/esxi"
 	_ "yunion.io/x/onecloud/pkg/multicloud/esxi/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -88,13 +93,28 @@ func newClient(options *BaseOptions) (*esxi.SESXiClient, error) {
 		return nil, fmt.Errorf("Missing password")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	return esxi.NewESXiClient2(
 		esxi.NewESXiClientConfig(
 			options.Host,
 			options.Port,
 			options.Account,
 			options.Password,
-		),
+		).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 }
 

cmd/fetcherfs/fs.go

@@ -28,7 +28,7 @@ import (
 	"syscall"
 
 	"bazil.org/fuse/fs"
-	"github.com/pierrec/lz4"
+	"github.com/pierrec/lz4/v4"
 	"github.com/pkg/errors"
 
 	"yunion.io/x/log"
@@ -227,6 +227,9 @@ func (fs *FetcherFs) doFetchData(idx int64) error {
 	}
 
 	var lz4Reader = lz4.NewReader(res.Body)
+	if err := lz4Reader.Apply(lz4.ConcurrencyOption(-1)); err != nil {
+		return errors.Errorf("Apply lz4 option: %v", err)
+	}
 
 	buf, err := ioutil.ReadAll(lz4Reader)
 	if len(buf) != int(end-start+1) {

cmd/fetcherfs/main.go

@@ -23,23 +23,40 @@ import (
 	"github.com/sevlyar/go-daemon"
 
 	"yunion.io/x/log"
+	"yunion.io/x/log/hooks"
 	"yunion.io/x/pkg/util/signalutils"
 )
 
 func main() {
-	cntxt := &daemon.Context{
-		WorkDir: "./",
-		Umask:   027,
+	if !opt.Foreground {
+		cntxt := &daemon.Context{
+			WorkDir: "./",
+			Umask:   027,
+		}
+
+		d, err := cntxt.Reborn()
+		if err != nil {
+			log.Fatalf("Unable to run: %s", err)
+		}
+		if d != nil {
+			return
+		}
+		defer cntxt.Release()
 	}
 
-	d, err := cntxt.Reborn()
-	if err != nil {
-		log.Fatalf("Unable to run: %s", err)
+	if opt.Debug {
+		logFileHook := hooks.LogFileRotateHook{
+			LogFileHook: hooks.LogFileHook{
+				FileDir:  "/tmp",
+				FileName: "fetcherfs.log",
+			},
+			RotateNum:  10,
+			RotateSize: 4096,
+		}
+		logFileHook.Init()
+		defer logFileHook.DeInit()
+		log.Logger().AddHook(&logFileHook)
 	}
-	if d != nil {
-		return
-	}
-	defer cntxt.Release()
 
 	fetcherFs, err := initFetcherFs()
 	if err != nil {
@@ -50,6 +67,9 @@ func main() {
 		opt.MountPoint,
 		fuse.FSName("fetcherfs"),
 		fuse.Subtype("fetcher"),
+		// https://github.com/bazil/fuse/issues/175
+		// fuse.MaxReadahead(128*1024),
+		fuse.MaxReadahead(uint32(opt.Blocksize*1024*1024)),
 	)
 	if err != nil {
 		log.Fatalln(err)

cmd/fetcherfs/options.go

@@ -30,6 +30,7 @@ type Options struct {
 	Blocksize  int    `help:"block size of content file system(MB)"`
 	MountPoint string `help:"mount path of fuse fs" required:"true"`
 	Debug      bool   `help:"enable debug go fuse"`
+	Foreground bool   `help:"run in foreground"`
 }
 
 var opt = &Options{}

cmd/googlecli/main.go

@@ -16,13 +16,18 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/google"
 	_ "yunion.io/x/onecloud/pkg/multicloud/google/shell"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
@@ -112,13 +117,28 @@ func newClient(options *BaseOptions) (*google.SRegion, error) {
 		return nil, fmt.Errorf("Missing ProjectID")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := google.NewGoogleClient(
 		google.NewGoogleClientConfig(
 			options.ProjectID,
 			options.ClientEmail,
 			options.PrivateKeyID,
 			options.PrivateKey,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/hcsocli/main.go

@@ -16,8 +16,12 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -87,13 +91,28 @@ func newClient(options *BaseOptions) (*huawei.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := huawei.NewHuaweiClient(
 		huawei.NewHuaweiClientConfig(
 			options.AccessKey,
 			options.Secret,
 			options.ProjectId,
 			&options.SHCSOEndpoints,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/huaweicli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/huawei"
 	_ "yunion.io/x/onecloud/pkg/multicloud/huawei/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -86,13 +91,28 @@ func newClient(options *BaseOptions) (*huawei.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := huawei.NewHuaweiClient(
 		huawei.NewHuaweiClientConfig(
 			options.CloudEnv,
 			options.AccessKey,
 			options.Secret,
 			options.ProjectId,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/jdcloudcli/main.go

@@ -16,13 +16,17 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
 	"github.com/sirupsen/logrus"
+	"golang.org/x/net/http/httpproxy"
 
 	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/jdcloud"
 	_ "yunion.io/x/onecloud/pkg/multicloud/jdcloud/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -89,7 +93,22 @@ func newClient(options *Options) (*jdcloud.SRegion, error) {
 	if regionId == "" {
 		regionId = jdcloud.JDCLOUD_DEFAULT_REGION
 	}
-	region := jdcloud.NewRegion(regionId, options.AccessKey, options.AccessSecret, nil, options.Debug)
+
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
+	cfcg := &cloudprovider.ProviderConfig{
+		ProxyFunc: proxyFunc,
+	}
+
+	region := jdcloud.NewRegion(regionId, options.AccessKey, options.AccessSecret, cfcg, options.Debug)
 	if region == nil {
 		return nil, fmt.Errorf("no such region %s", regionId)
 	}

cmd/openstackcli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/openstack"
 	_ "yunion.io/x/onecloud/pkg/multicloud/openstack/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -93,6 +98,16 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := openstack.NewOpenStackClient(
 		openstack.NewOpenstackClientConfig(
 			options.AuthURL,
@@ -103,7 +118,12 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) {
 		).
 			EndpointType(options.EndpointType).
 			DomainName(options.DomainName).
-			Debug(options.Debug),
+			Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/qcloudcli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/qcloud"
 	_ "yunion.io/x/onecloud/pkg/multicloud/qcloud/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -85,11 +90,27 @@ func newClient(options *BaseOptions) (*qcloud.SRegion, error) {
 		return nil, fmt.Errorf("Missing SecretID")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	if cli, err := qcloud.NewQcloudClient(
 		qcloud.NewQcloudClientConfig(
 			options.SecretID,
 			options.SecretKey,
-		).AppId(options.AppID).Debug(options.Debug),
+		).AppId(options.AppID).
+			Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	); err != nil {
 		return nil, err
 	} else if region := cli.GetRegion(options.RegionId); region == nil {

cmd/rbdcli/main.go

@@ -1,130 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"yunion.io/x/pkg/errors"
-	"yunion.io/x/structarg"
-
-	_ "yunion.io/x/onecloud/cmd/rbdcli/shell"
-	"yunion.io/x/onecloud/pkg/util/rbdutils"
-	"yunion.io/x/onecloud/pkg/util/shellutils"
-)
-
-type BaseOptions struct {
-	Help       bool   `help:"Show help"`
-	Debug      bool   `help:"debug mode"`
-	MonHost    string `help:"Ceph Mon Host" default:"$MON_HOST"`
-	Pool       string `help:"Ceph pool" default:"$CEPH_POOL" metavar:"CEPH_POOL"`
-	Key        string `help:"Secret" default:"$CEPH_KEY" metavar:"CEPH_KEY"`
-	SUBCOMMAND string `help:"rbdcli subcommand" subcommand:"true"`
-}
-
-func getSubcommandParser() (*structarg.ArgumentParser, error) {
-	parse, e := structarg.NewArgumentParser(&BaseOptions{},
-		"rbdcli",
-		"Command-line interface to rbd API.",
-		`See "rbdcli help COMMAND" for help on a specific command.`)
-
-	if e != nil {
-		return nil, e
-	}
-
-	subcmd := parse.GetSubcommand()
-	if subcmd == nil {
-		return nil, fmt.Errorf("No subcommand argument.")
-	}
-	type HelpOptions struct {
-		SUBCOMMAND string `help:"sub-command name"`
-	}
-	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
-		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
-		if e != nil {
-			return e
-		} else {
-			fmt.Print(helpstr)
-			return nil
-		}
-	})
-	for _, v := range shellutils.CommandTable {
-		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
-		if e != nil {
-			return nil, e
-		}
-	}
-	return parse, nil
-}
-
-func showErrorAndExit(e error) {
-	fmt.Fprintf(os.Stderr, "%s", e)
-	fmt.Fprintln(os.Stderr)
-	os.Exit(1)
-}
-
-func newPool(opts *BaseOptions) (*rbdutils.SPool, error) {
-	cli, err := rbdutils.NewCluster(opts.MonHost, opts.Key)
-	if err != nil {
-		return nil, err
-	}
-
-	pool, err := cli.GetPool(opts.Pool)
-	if err != nil {
-		return nil, errors.Wrapf(err, "GetPool(%s)", opts.Pool)
-	}
-
-	return pool, nil
-}
-
-func main() {
-	parser, e := getSubcommandParser()
-	if e != nil {
-		showErrorAndExit(e)
-	}
-	e = parser.ParseArgs(os.Args[1:], false)
-	options := parser.Options().(*BaseOptions)
-
-	if options.Help {
-		fmt.Print(parser.HelpString())
-		return
-	}
-	subcmd := parser.GetSubcommand()
-	subparser := subcmd.GetSubParser()
-	if e != nil {
-		if subparser != nil {
-			fmt.Print(subparser.Usage())
-		} else {
-			fmt.Print(parser.Usage())
-		}
-		showErrorAndExit(e)
-		return
-	}
-	suboptions := subparser.Options()
-	if options.SUBCOMMAND == "help" {
-		e = subcmd.Invoke(suboptions)
-	} else {
-		var pool *rbdutils.SPool
-		pool, e = newPool(options)
-		if e != nil {
-			showErrorAndExit(e)
-		}
-		e = subcmd.Invoke(pool, suboptions)
-	}
-	if e != nil {
-		showErrorAndExit(e)
-	}
-}

cmd/rbdcli/shell/pool.go

@@ -1,58 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package shell
-
-import (
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
-
-	"yunion.io/x/onecloud/pkg/util/rbdutils"
-	"yunion.io/x/onecloud/pkg/util/shellutils"
-)
-
-func init() {
-	type PoolListOptions struct {
-	}
-	shellutils.R(&PoolListOptions{}, "pool-list", "List pools", func(cli *rbdutils.SPool, args *PoolListOptions) error {
-		cluster := cli.GetCluster()
-		pools, err := cluster.ListPools()
-		if err != nil {
-			return errors.Wrapf(err, "ListPools")
-		}
-		printObject(jsonutils.Marshal(map[string][]string{"pools": pools}))
-		return nil
-	})
-
-	type PoolDeleteOptions struct {
-		POOL string
-	}
-
-	shellutils.R(&PoolDeleteOptions{}, "pool-delete", "Delete Cluster pool", func(cli *rbdutils.SPool, args *PoolDeleteOptions) error {
-		return cli.GetCluster().DeletePool(args.POOL)
-	})
-
-	type ImageListOptions struct {
-	}
-
-	shellutils.R(&ImageListOptions{}, "image-list", "List Pool images", func(cli *rbdutils.SPool, args *ImageListOptions) error {
-		images, err := cli.ListImages()
-		if err != nil {
-			return err
-		}
-		printObject(jsonutils.Marshal(map[string][]string{"images": images}))
-		return nil
-	})
-
-}

cmd/ucloudcli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/ucloud"
 	_ "yunion.io/x/onecloud/pkg/multicloud/ucloud/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -86,11 +91,26 @@ func newClient(options *BaseOptions) (*ucloud.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := ucloud.NewUcloudClient(
 		ucloud.NewUcloudClientConfig(
 			options.AccessKey,
 			options.Secret,
-		).ProjectId(options.ProjectId).Debug(options.Debug),
+		).ProjectId(options.ProjectId).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

cmd/vpcagent/main.go

@@ -23,6 +23,7 @@ import (
 
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/appctx"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/util/atexit"
@@ -60,6 +61,7 @@ func main() {
 
 		wg := &sync.WaitGroup{}
 		ctx = context.WithValue(ctx, "wg", wg)
+		ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "vpcagent")
 		wg.Add(1)
 		go w.Start(ctx)
 

cmd/zstackcli/main.go

@@ -16,10 +16,15 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/structarg"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/zstack"
 	_ "yunion.io/x/onecloud/pkg/multicloud/zstack/shell"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -89,12 +94,27 @@ func newClient(options *BaseOptions) (*zstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
+	cfg := &httpproxy.Config{
+		HTTPProxy:  os.Getenv("HTTP_PROXY"),
+		HTTPSProxy: os.Getenv("HTTPS_PROXY"),
+		NoProxy:    os.Getenv("NO_PROXY"),
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+
 	cli, err := zstack.NewZStackClient(
 		zstack.NewZstackClientConfig(
 			options.AuthURL,
 			options.Username,
 			options.Password,
-		).Debug(options.Debug),
+		).Debug(options.Debug).
+			CloudproviderConfig(
+				cloudprovider.ProviderConfig{
+					ProxyFunc: proxyFunc,
+				},
+			),
 	)
 	if err != nil {
 		return nil, err

go.mod

@@ -36,7 +36,6 @@ require (
 	github.com/bitly/go-simplejson v0.5.0
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/c-bata/go-prompt v0.2.1
-	github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94
 	github.com/coredns/coredns v1.3.0
 	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
@@ -48,19 +47,19 @@ require (
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/farsightsec/golang-framestream v0.0.0-20181102145529-8a0cb8ba8710 // indirect
-	github.com/fatih/color v1.7.0
+	github.com/fatih/color v1.10.0
 	github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/ghodss/yaml v1.0.0
 	github.com/gin-gonic/gin v1.7.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
-	github.com/go-ini/ini v1.62.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-yaml/yaml v2.1.0+incompatible
-	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/gofrs/uuid v4.1.0+incompatible // indirect
 	github.com/golang-plus/errors v1.0.0
 	github.com/golang-plus/testing v1.0.0 // indirect
 	github.com/golang-plus/uuid v1.0.0
@@ -84,7 +83,7 @@ require (
 	github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect
 	github.com/libvirt/libvirt-go-xml v5.2.0+incompatible
 	github.com/ma314smith/signedxml v0.0.0-20200410192636-c342a2d0ae60
-	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/mattn/go-runewidth v0.0.12 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/mattn/go-tty v0.0.0-20181127064339-e4f871175a2f // indirect
 	github.com/mdlayher/arp v0.0.0-20190313224443-98a83c8a2717
@@ -100,11 +99,13 @@ require (
 	github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
 	github.com/opentracing/opentracing-go v1.0.2 // indirect
 	github.com/openzipkin/zipkin-go-opentracing v0.3.4 // indirect
-	github.com/pierrec/lz4 v2.0.5+incompatible
+	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/pierrec/lz4/v4 v4.1.12
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/term v0.0.0-20181116001808-27bbf2edb814 // indirect
 	github.com/pquerna/otp v1.2.0
 	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/serialx/hashring v0.0.0-20180504054112-49a4782e9908
 	github.com/sevlyar/go-daemon v0.1.5
@@ -131,7 +132,7 @@ require (
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57
 	golang.org/x/text v0.3.3
 	golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 // indirect
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
@@ -149,14 +150,12 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
-	yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349
+	yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
+	yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f
+	yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada
+	yunion.io/x/sqlchemy v1.0.0
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
-
-replace github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 => github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859

go.sum

@@ -172,6 +172,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -211,12 +212,15 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/farsightsec/golang-framestream v0.0.0-20181102145529-8a0cb8ba8710 h1:QdyRyGZWLEvJG5Kw3VcVJvhXJ5tZ1MkRgqpJOEZSySM=
 github.com/farsightsec/golang-framestream v0.0.0-20181102145529-8a0cb8ba8710/go.mod h1:eNde4IQyEiA5br02AouhEHCu3p3UzrCdFR4LuQHklMI=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e h1:P10tZmVD2XclAaT9l7OduMH1OLFzTa1wUuUqHZnEdI0=
 github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -235,8 +239,6 @@ github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
-github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
@@ -263,8 +265,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
-github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
-github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.1.0+incompatible h1:sIa2eCvUTwgjbqXrPLfNwUf9S3i3mpH1O1atV+iL/Wk=
+github.com/gofrs/uuid v4.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
@@ -306,8 +308,9 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -403,11 +406,13 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911 h1:FvnrqecqX4zT0wOIbYK1gNgTm0677INEWiFY8UEYggY=
@@ -426,16 +431,17 @@ github.com/ma314smith/signedxml v0.0.0-20200410192636-c342a2d0ae60 h1:q5rqPuvxdO
 github.com/ma314smith/signedxml v0.0.0-20200410192636-c342a2d0ae60/go.mod h1:KEgVcb43+f5KFUH/x6Vd3NROG0AIL2CuKMrIqYsmx6E=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-sqlite3 v1.7.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -511,8 +517,10 @@ github.com/openzipkin/zipkin-go-opentracing v0.3.4/go.mod h1:js2AbwmHW0YD9DwIw2J
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
-github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
+github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8=
+github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -537,8 +545,13 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
@@ -623,8 +636,6 @@ github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0s
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859 h1:wu596gn6sV3j5wy+GDfiG8nwtzIDpalJihfmC/TjoYc=
-github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859/go.mod h1:8XuBae5AzsgotLArJSewMruYVaQs8AlfsK5jBCG8T9Y=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko=
@@ -752,10 +763,12 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -909,11 +922,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
-yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
+yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 h1:ESeezAb9LM2Dcy28DDrD2nI+h22qEf4MyAPXwuDO4uM=
-yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e h1:zOVxZ0HrDOHLTm5YxkDKUm1x5jMVKbQ/iBSAyn5l1+Q=
+yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o=
@@ -923,11 +936,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f h1:maply0f3AsvoXqkpTKkb6/tZFbRAT/rdkk1Cw3FkoB8=
-yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg=
+yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada h1:ZIhSSwFHfnzPMKSyiN0nkGwvC6RUeuCIAnWuTtDS900=
-yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v1.0.0 h1:fRtWxe1xxMR+NDVKYNuI89m2HGSl6aad2DZ+Bn/mZ4Q=
+yunion.io/x/sqlchemy v1.0.0/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -185,7 +185,22 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 
 	resp.Add(jsonutils.NewArray(retIdps...), "idps")
 
-	resp.Add(jsonutils.NewString(options.Options.ApiServer), "api_server")
+	// in case api_server changed but not synchronized from Keystone
+	// fetch this option directly from Keystone
+	var apiSrv string
+	commonCfg, err := modules.ServicesV3.GetSpecific(s, "common", "config", nil)
+	if err != nil {
+		log.Errorf("fetch common config error: %s", err)
+	} else {
+		apiSrv, err = commonCfg.GetString("config", "default", "api_server")
+		if err != nil {
+			log.Errorf("fetch api_server fail: %s (cfg: %s)", err, commonCfg)
+		}
+	}
+	if len(apiSrv) == 0 {
+		apiSrv = options.Options.ApiServer
+	}
+	resp.Add(jsonutils.NewString(apiSrv), "api_server")
 
 	return resp, nil
 }

pkg/apigateway/handler/misc.go

@@ -346,8 +346,9 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 		rowIdx := i + 2
 		name := row[0]
 		password := row[1]
-		domain := row[2]
-		allowWebConsole := strings.ToLower(row[3])
+		displayname := row[2]
+		domain := row[3]
+		allowWebConsole := strings.ToLower(row[4])
 
 		// 忽略空白行
 		rowStr := strings.Join(row, "")
@@ -361,6 +362,10 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 			return
 		}
 
+		if len(displayname) == 0 {
+			displayname = name
+		}
+
 		if len(password) == 0 {
 			e := httperrors.NewClientError("row %d password is empty", rowIdx)
 			httperrors.JsonClientError(ctx, w, e)
@@ -401,6 +406,7 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 
 		user := jsonutils.NewDict()
 		user.Add(jsonutils.NewString(name), "name")
+		user.Add(jsonutils.NewString(displayname), "displayname")
 		user.Add(jsonutils.NewString(domainId), "domain_id")
 		if len(password) > 0 {
 			user.Add(jsonutils.NewString(password), "password")
@@ -468,7 +474,7 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 			return
 		}
 	case "BatchUserRegister":
-		records := [][]string{{"*用户名（user）", "*用户密码（password）", "*部门/域（domain）", "*是否登录控制台（allow_web_console：true、false）"}}
+		records := [][]string{{"*用户名（user）", "*用户密码（password）", "*显示名（displayname）", "*部门/域（domain）", "*是否登录控制台（allow_web_console：true、false）"}}
 		content, err = writeXlsx("users", records)
 		if err != nil {
 			httperrors.InternalServerError(ctx, w, "internal server error")

pkg/apigateway/handler/resource.go

@@ -192,9 +192,10 @@ func (f *ResourceHandlers) doList(ctx context.Context, session *mcclient.ClientS
 	var exportKeys []string
 	var exportTexts []string
 	export := struct {
-		ExportFormat string `json:"export"`
-		ExportKeys   string `json:"export_keys"`
-		ExportTexts  string `json:"export_texts"`
+		ExportFormat   string `json:"export"`
+		ExportFileName string `json:"export_file_name"`
+		ExportKeys     string `json:"export_keys"`
+		ExportTexts    string `json:"export_texts"`
 	}{}
 	query.Unmarshal(&export)
 	if len(export.ExportFormat) > 0 {
@@ -218,6 +219,7 @@ func (f *ResourceHandlers) doList(ctx context.Context, session *mcclient.ClientS
 		queryDict := query.(*jsonutils.JSONDict)
 		queryDict.Remove("export")
 		queryDict.Remove("export_texts")
+		queryDict.Remove("export_file_name")
 		keys := []string{}
 		for _, key := range exportKeys {
 			if !strings.Contains(key, ".") {
@@ -242,7 +244,11 @@ func (f *ResourceHandlers) doList(ctx context.Context, session *mcclient.ClientS
 		w.Header().Set("Content-Description", "File Transfer")
 		w.Header().Set("Content-Transfer-Encoding", "binary")
 		w.Header().Set("Content-Type", "application/octet-stream")
-		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"export-%s.xlsx\"", module.KeyString()))
+		fileName := fmt.Sprintf("export-%s", module.KeyString())
+		if len(export.ExportFileName) > 0 {
+			fileName = export.ExportFileName
+		}
+		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.xlsx\"", fileName))
 		excelutils.Export(ret.Data, exportKeys, exportTexts, w)
 		return
 	}

pkg/apigateway/options/options.go

@@ -26,7 +26,7 @@ type GatewayOptions struct {
 
 	DisableModuleApiVersion bool `help:"Disable each modules default api version" default:"false"`
 
-	EnableTotp bool `help:"Enable two-factor authentication" default:"true"`
+	EnableTotp bool `help:"Enable two-factor authentication" default:"false"`
 
 	SsoRedirectUrl     string `help:"SSO idp redirect URL"`
 	SsoAuthCallbackUrl string `help:"SSO idp auth callback URL"`

pkg/apihelper/apihelper.go

@@ -118,7 +118,7 @@ func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess
 	if s != nil {
 		token := s.GetToken()
 		expires := token.GetExpires()
-		if time.Now().Add(time.Hour).After(expires) {
+		if time.Now().Add(time.Hour).Before(expires) {
 			return s
 		}
 	}

pkg/apihelper/reflect.go

@@ -102,6 +102,7 @@ func GetModels(opts *GetModelsOptions) error {
 	}
 
 	listOptions := options.BaseListOptions{
+		System:       options.Bool(true),
 		Admin:        options.Bool(true),
 		Details:      options.Bool(opts.IncludeDetails),
 		ShowEmulated: options.Bool(opts.IncludeEmulated),

pkg/apis/cloudcommon/db/metadata.go

@@ -14,8 +14,10 @@
 
 package db
 
+import "yunion.io/x/onecloud/pkg/apis"
+
 const (
-	CLOUD_TAG_PREFIX     = "ext:"
-	USER_TAG_PREFIX      = "user:"
-	SYS_CLOUD_TAG_PREFIX = "sys:"
+	CLOUD_TAG_PREFIX     = apis.CLOUD_TAG_PREFIX
+	USER_TAG_PREFIX      = apis.USER_TAG_PREFIX
+	SYS_CLOUD_TAG_PREFIX = apis.SYS_CLOUD_TAG_PREFIX
 )

pkg/apis/cloudproxy/proxy_endpoints.go

@@ -30,7 +30,8 @@ type ProxyEndpointCreateInput struct {
 }
 
 type ProxyEndpointCreateFromServerInput struct {
-	Name string
+	Name         string
+	GenerateName string
 
 	ServerId string
 }

pkg/apis/compute/api.go

@@ -396,6 +396,10 @@ type ServerCreateInput struct {
 	// emulate: BIOS, UEFI
 	Bios string `json:"bios"`
 
+	// Machine类型
+	// emulate: pc, q35
+	Machine string `json:"machine"`
+
 	// 启动顺序
 	// c: cdrome
 	// d: disk
@@ -551,14 +555,16 @@ type GuestBatchMigrateRequest struct {
 	PreferHostId string `json:"prefer_host_id"`
 	// Deprecated
 	// swagger:ignore
-	PreferHost string `json:"prefer_host" yunion-deprecated-by:"prefer_host_id"`
+	PreferHost   string `json:"prefer_host" yunion-deprecated-by:"prefer_host_id"`
+	SkipCpuCheck bool   `json:"skip_cpu_check"`
 }
 
 type GuestBatchMigrateParams struct {
-	Id          string
-	LiveMigrate bool
-	RescueMode  bool
-	OldStatus   string
+	Id           string
+	LiveMigrate  bool
+	SkipCpuCheck bool
+	RescueMode   bool
+	OldStatus    string
 }
 
 type HostLoginInfo struct {

pkg/apis/compute/cdn.go

@@ -14,7 +14,10 @@
 
 package compute
 
-import "yunion.io/x/onecloud/pkg/apis"
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
 
 const (
 	CDN_DOMAIN_STATUS_ONLINE        = "online"
@@ -25,12 +28,18 @@ const (
 	CDN_DOMAIN_STATUS_REJECTED      = "rejected"
 	CDN_DOMAIN_STATUS_UNKNOWN       = "unknown"
 
-	CDN_DOMAIN_AREA_MAINLAND      = "mainland"
-	CDN_DOMAIN_AREA_OVERSEAS      = "overseas"
-	CDN_DOMAIN_AREA_GLOBAL        = "global"
-	CDN_DOMAIN_ORIGIN_TYPE_DOMAIN = "domain"
-	CDN_DOMAIN_ORIGIN_TYPE_IP     = "ip"
-	CDN_DOMAIN_ORIGIN_TYPE_BUCKET = "bucket"
+	CDN_DOMAIN_AREA_MAINLAND       = "mainland"
+	CDN_DOMAIN_AREA_OVERSEAS       = "overseas"
+	CDN_DOMAIN_AREA_GLOBAL         = "global"
+	CDN_DOMAIN_ORIGIN_TYPE_DOMAIN  = "domain"
+	CDN_DOMAIN_ORIGIN_TYPE_IP      = "ip"
+	CDN_DOMAIN_ORIGIN_TYPE_BUCKET  = "bucket"
+	CDN_DOMAIN_ORIGIN_THIRED_PARTY = "third_party"
+
+	// Qcloud
+	CDN_SERVICE_TYPE_WEB      = "web"      // 静态加速
+	CND_SERVICE_TYPE_DOWNLOAD = "download" // 下载加速
+	CND_SERVICE_TYPE_MEDIA    = "media"    // 流媒体点播加速
 )
 
 type CdnDomain struct {
@@ -53,6 +62,23 @@ type CdnDomains struct {
 }
 
 type CDNDomainCreateInput struct {
+	apis.EnabledStatusInfrasResourceBaseCreateInput
+
+	// 源站信息
+	// required: true
+	Origins *cloudprovider.SCdnOrigins
+
+	// 服务类型
+	// required: true
+	// enmu: web, download, media
+	ServiceType string `json:"service_type"`
+	// 加速区域
+	// enmu: mainland, overseas, global
+	// requrired: true
+	Area string `json:"area"`
+
+	CloudproviderResourceInput
+	DeletePreventableCreateInput
 }
 
 type CDNDomainDetails struct {

pkg/apis/compute/cloudproviderregion.go

@@ -39,6 +39,9 @@ type CloudproviderregionDetails struct {
 	// example: 300
 	SyncIntervalSeconds int `json:"sync_interval_seconds"`
 
+	// 云订阅同步状态
+	CloudproviderSyncStatus string `json:"cloudprovider_sync_status"`
+
 	// 支持服务列表
 	Capabilities []string `json:"capabilities"`
 }

pkg/apis/compute/cloudregions_const.go

@@ -67,6 +67,7 @@ const (
 	CITY_XIA_MEN        = "XiaMen"       //厦门
 	CITY_HUAI_NAN       = "HuaiNan"      //淮南
 	CITY_SU_QIAN        = "SuQian"       //宿迁
+	CITY_DA_LIAN        = "Dalian"       //大连
 
 	// 日本
 	CITY_TOKYO = "Tokyo" //东京
@@ -101,6 +102,7 @@ const (
 	CITY_JAKARTA = "Jakarta" //雅加达
 
 	// 印度
+	CITY_DELHI       = "Delhi"       // 德里
 	CITY_MUMBAI      = "Mumbai"      //孟买
 	CITY_KANCHIPURAM = "Kanchipuram" //甘吉布勒姆
 	CITY_MAHARASHTRA = "Maharashtra" //马哈拉施特拉邦
@@ -159,6 +161,9 @@ const (
 	// 瑞典
 	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
 
+	// 波兰
+	CITY_WARSAW = "Warsaw" // 华沙
+
 	// 巴西
 	CITY_SAO_PAULO      = "Sao Paulo"      //圣保罗
 	CITY_RIO_DE_JANEIRO = "Rio de Janeiro" // 里约热内卢
@@ -220,4 +225,7 @@ const (
 	COUNTRY_CODE_MX = "MX" //墨西哥
 	COUNTRY_CODE_CL = "CL" //智利
 	COUNTRY_CODE_BH = "BH" //巴林
+	COUNTRY_CODE_PL = "PL" //波兰
+	COUNTRY_CODE_FI = "FI" //芬兰
+	COUNTRY_CODE_BE = "BE" //比利时
 )

pkg/apis/compute/disk.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/apis/billing"
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter"
 )
 
 type DiskCreateInput struct {
@@ -265,3 +266,24 @@ func (self DiskResizeInput) SizeMb() (int, error) {
 	}
 	return fileutils.GetSizeMb(self.Size, 'M', 1024)
 }
+
+type DiskAllocateInput struct {
+	Format        string
+	DiskSizeMb    int
+	ImageId       string
+	FsFormat      string
+	Encryption    bool
+	Rebuild       bool
+	BackingDiskId string
+	SnapshotId    string
+
+	SnapshotUrl        string
+	SnapshotOutOfChain bool
+	Protocol           string
+	SrcDiskId          string
+	SrcPool            string
+
+	// vmware
+	HostIp    string
+	Datastore vcenter.SVCenterAccessInfo
+}

pkg/apis/compute/disk_const.go

@@ -43,6 +43,9 @@ const (
 	DISK_POST_MIGRATE  = "post_migrate"
 	DISK_MIGRATING     = "migrating"
 
+	DISK_CLONE      = "clone"
+	DISK_CLONE_FAIL = "clone_failed"
+
 	DISK_START_SNAPSHOT       = "start_snapshot"
 	DISK_SNAPSHOTING          = "snapshoting"
 	DISK_APPLY_SNAPSHOT_FAIL  = "apply_snapshot_failed"

pkg/apis/compute/elastic_search.go

@@ -49,3 +49,6 @@ type ElasticSearchListInput struct {
 	ManagedResourceListInput
 	VpcFilterListInput
 }
+
+type ElasticSearchAccessInfoInput struct {
+}

pkg/apis/compute/elasticip.go

@@ -103,4 +103,14 @@ type ElasticipAssociateInput struct {
 	// enmu: server, natgateway
 	// default: server
 	InstanceType string `json:"instance_type"`
+
+	// EIP映射的内网IP地址，可选
+	IpAddr string `json:"ip_addr"`
+}
+
+type ElasticDissociateInput struct {
+	// 注意: 若关联到aws NAT网关后，目前没办法解除关联关系
+	// 是否解绑后自动删除弹性公网IP
+	// default: false
+	AutoDelete bool `json:"auto_delete"`
 }

pkg/apis/compute/elasticips_const.go

@@ -27,6 +27,8 @@ const (
 	EIP_ASSOCIATE_TYPE_LOADBALANCER = "loadbalancer"
 	EIP_ASSOCIATE_TYPE_UNKNOWN      = "unknown"
 
+	EIP_ASSOCIATE_TYPE_INSTANCE_GROUP = "instancegroup"
+
 	EIP_STATUS_READY           = "ready"
 	EIP_STATUS_UNKNOWN         = "unknown"
 	EIP_STATUS_ALLOCATE        = "allocate"