Merge pull request #10614 from swordqiu/automated-cherry-pick-of-#10612-upstream-release-3.6

Automated cherry pick of #10612: fix(region): add usage cache, speedup response

Makefile

@@ -134,6 +134,11 @@ clean:
 fmt:
 	@git ls-files --exclude '*' '*.go' \
 		| grep -v '^vendor/' \
+		| while read f; do \
+			if ! grep -m1 -q '^// Code generated .* DO NOT EDIT\.$$' "$$f"; then \
+				echo "$$f"; \
+			fi ; \
+		done \
 		| xargs $(XARGS_FLAGS) gofmt -w
 
 fmt-check: fmt
@@ -206,70 +211,6 @@ mod:
 	go mod tidy
 	go mod vendor -v
 
-
-DOCKER_CENTOS_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-3
-
-define dockerCentOSBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
-docker-centos-build:
-	docker rm --force onecloud-ci-build &>/dev/null || true
-	docker run \
-		--name onecloud-docker-centos-build \
-		--rm \
-		--volume $(CURDIR):/root/onecloud \
-		--volume $(CURDIR)/_output/_cache:/root/.cache \
-		$(DOCKER_CENTOS_BUILD_IMAGE) \
-		/bin/bash -c "$$dockerCentOSBuildCmd"
-	chown -R $$(id -u):$$(id -g) _output
-	ls -lh _output/bin
-
-# NOTE we need a way to stop and remove the container started by docker-build.
-# No --tty, --stop-signal won't work
-docker-centos-build-stop:
-	docker stop --time 0 onecloud-docker-centos-build || true
-
-.PHONY: docker-centos-build
-.PHONY: docker-centos-build-stop
-
-DOCKER_ALPINE_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5
-
-define dockerAlpineBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/go/src/yunion.io/x/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
-docker-alpine-build:
-	docker rm --force onecloud-docker-alpine-build &>/dev/null || true
-	docker run --rm \
-		--name onecloud-docker-alpine-build \
-		-v $(CURDIR):/root/go/src/yunion.io/x/onecloud \
-		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
-		$(DOCKER_ALPINE_BUILD_IMAGE) \
-		/bin/sh -c "$$dockerAlpineBuildCmd"
-	ls -lh _output/alpine-build/bin
-
-docker-alpine-build-stop:
-	docker stop --time 0 onecloud-docker-alpine-build || true
-
-.PHONY: docker-alpine-build
-.PHONY: docker-alpine-build-stop
-
 define helpText
 Build with docker
 
@@ -329,3 +270,6 @@ image:
 
 %:
 	@:
+
+ModName:=yunion.io/x/onecloud
+include $(CURDIR)/Makefile.common.mk

Makefile.common.mk

@@ -0,0 +1,83 @@
+ifeq ($(__inc_Makefile_common_mk),)
+__inc_Makefile_common_mk:=1
+
+ifeq ($(ModName),)
+  $(error ModName must be set, e.g. yunion.io/x/onecloud)
+endif
+ModBaseName:=$(notdir $(ModName))
+
+DockerImageRegistry?=registry.cn-beijing.aliyuncs.com
+DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.0-5
+DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-4
+
+
+EnvIf=$(if $($(1)),$(1)=$($(1)))
+
+define dockerCentOSBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
+docker-centos-build:
+	docker rm --force docker-centos-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-centos-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/centos-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/centos-build/_cache:/root/.cache \
+		$(DockerImageCentOSBuild) \
+		/bin/bash -c "$$dockerCentOSBuildCmd"
+	ls -lh _output/centos-build/bin
+
+# NOTE we need a way to stop and remove the container started by docker-build.
+# No --tty, --stop-signal won't work
+docker-centos-build-stop:
+	docker stop --time 0 docker-centos-build-$(ModBaseName) || true
+
+.PHONY: docker-centos-build
+.PHONY: docker-centos-build-stop
+
+
+define dockerAlpineBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
+docker-alpine-build:
+	docker rm --force docker-alpine-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-alpine-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
+		$(DockerImageAlpineBuild) \
+		/bin/sh -c "$$dockerAlpineBuildCmd"
+	ls -lh _output/alpine-build/bin
+
+docker-alpine-build-stop:
+	docker stop --time 0 docker-alpine-build-$(ModBaseName) || true
+
+.PHONY: docker-alpine-build
+.PHONY: docker-alpine-build-stop
+
+endif # __inc_Makefile_common_mk

build/docker/Dockerfile.baremetal-base

@@ -1,3 +1,6 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
+RUN yum install -y https://iso.yunion.cn/3.6/rpms/packages/host/baremetal-pxerom-1.1.0-20072810.x86_64.rpm
+
 FROM frolvlad/alpine-glibc:glibc-2.28
 
 MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
@@ -16,17 +19,17 @@ RUN mkdir -p /opt/cloud/yunion/baremetal
 RUN wget -O /opt/cloud/yunion/baremetal/pci.ids http://pci-ids.ucw.cz/v2.2/pci.ids
 
 # download baremetal-pxerom-*.x86_64.rpm and unpack rpm in current directory
-COPY ./opt/cloud/yunion/baremetal/bootia32.efi /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/bootx64.efi /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/chain.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/initramfs /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/isolinux.bin /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/kernel /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.e32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.e64 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/libcom32.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/bootia32.efi /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/bootx64.efi /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/chain.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/initramfs /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/isolinux.bin /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/kernel /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.e32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.e64 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/libcom32.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal

build/docker/Dockerfile.host-deployer

@@ -6,4 +6,4 @@ ENV TZ Asia/Shanghai
 
 RUN mkdir -p /opt/yunion/bin
 
-ADD ./_output/alpine-build/bin/host-deployer /opt/yunion/bin/host-deployer
+ADD ./_output/centos-build/bin/host-deployer /opt/yunion/bin/host-deployer

build/docker/Dockerfile.telegraf-raid

@@ -1,3 +0,0 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
-
-ADD ./_output/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/Dockerfile.telegraf-raid-plugin

@@ -0,0 +1,3 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+
+ADD ./_output/centos-build/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/multi-arch/Dockerfile.host.dockerignore

@@ -0,0 +1,2 @@
+_output/alpine-build/_cache
+

cmd/climc/shell/compute/buckets.go

@@ -452,11 +452,15 @@ func init() {
 		DomainList []string
 		// 是否允许空referer 访问
 		AllowEmptyRefer bool `help:"all empty refer access"`
+		Enabled         bool
+		RerererType     string `help:"Referer type" choices:"Black-List|White-List"`
 	}
 	R(&BucketSetRefererOption{}, "bucket-set-referer", "Set bucket referer", func(s *mcclient.ClientSession, args *BucketSetRefererOption) error {
 		conf := api.BucketRefererConf{
-			WhiteList:       args.DomainList,
+			Enabled:         args.Enabled,
 			AllowEmptyRefer: args.AllowEmptyRefer,
+			RefererType:     args.RerererType,
+			DomainList:      args.DomainList,
 		}
 		result, err := modules.Buckets.PerformAction(s, args.ID, "set-referer", jsonutils.Marshal(conf))
 		if err != nil {

cmd/climc/shell/compute/usages.go

@@ -32,6 +32,8 @@ type GeneralUsageOptions struct {
 
 	CloudEnv string `help:"show usage of specified cloudenv" choices:"public|private|onpremise"`
 	Scope    string `help:"show usage of specified privilege scope" choices:"system|domain|project"`
+
+	Refresh bool `help:"force refresh usage statistics"`
 }
 
 func fetchHostTypeOptions(args *GeneralUsageOptions) *jsonutils.JSONDict {
@@ -48,6 +50,9 @@ func fetchHostTypeOptions(args *GeneralUsageOptions) *jsonutils.JSONDict {
 	if len(args.CloudEnv) > 0 {
 		params.Add(jsonutils.NewString(args.CloudEnv), "cloud_env")
 	}
+	if args.Refresh {
+		params.Add(jsonutils.JSONTrue, "refresh")
+	}
 	return params
 }
 

cmd/climc/shell/misc/copyright.go

@@ -36,6 +36,8 @@ func init() {
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
 		License   *string `help:"the license generator website address"`
+		BrandCn   *string `help:"the brand name in Chinese"`
+		BrandEn   *string `help:"the brand name in English"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -52,6 +54,14 @@ func init() {
 			params.Add(jsonutils.NewString(*args.License), "license")
 		}
 
+		if args.BrandCn != nil {
+			params.Add(jsonutils.NewString(*args.BrandCn), "brand_cn")
+		}
+
+		if args.BrandEn != nil {
+			params.Add(jsonutils.NewString(*args.BrandEn), "brand_en")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

go.mod

@@ -109,7 +109,7 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
-	github.com/tencentyun/cos-go-sdk-v5 v0.7.10
+	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
@@ -140,13 +140,13 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e
+	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
 	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2
+	yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210204013753-dbac29c9cedb
+	yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -583,8 +583,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
-github.com/tencentyun/cos-go-sdk-v5 v0.7.10 h1:hJPLdR4RPsnubB3M9qXNBeUbcOJMVANnO/L6Att/PZU=
-github.com/tencentyun/cos-go-sdk-v5 v0.7.10/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.24/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -912,8 +912,8 @@ sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e h1:NRtqiegW4NME0v10rpChORZnJnuAUbpv31R5DsqqczQ=
-yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
+yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
 yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
 yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
@@ -925,13 +925,12 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 h1:gjqNa2uQzIEXyySsbvVz0Ult
 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2 h1:NeCr2J8HjcIuJvEhP0rwWA1UKP8ReOv6HVf5k9YPtyA=
-yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c h1:J/joqbA1N2mAlOl0Uqd4LpAq3+DK5aoFMdz+p9Ld7pQ=
+yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210204013753-dbac29c9cedb h1:k7s5xMCd/fw29vUaNsGCrLkyn7w8eSghJ5NChuU9SAk=
-yunion.io/x/sqlchemy v0.0.0-20210204013753-dbac29c9cedb/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3 h1:C4yasW3gXronDIlIVUqHSbkA38PM1X9gxUWTYGsuIuI=
+yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/bucket.go

@@ -320,12 +320,16 @@ func (input *BucketCORSRules) Validate() error {
 }
 
 type BucketRefererConf struct {
-	// 白名单域名列表
-	WhiteList []string
-	// 黑名单域名列表
-	BlackList []string
+	// Referer Type
+	// enmu: Black-List, White-List
+	RefererType string
+	// 域名列表
+	DomainList []string
 	// 是否允许空referer 访问
 	AllowEmptyRefer bool
+
+	// 是否开启
+	Enabled bool `json:"enabled"`
 }
 
 func (input *BucketRefererConf) Validate() error {

pkg/apis/compute/cloudaccount.go

@@ -335,6 +335,15 @@ type CloudaccountPerformPublicInput struct {
 type CloudaccountPerformPrepareNetsInput struct {
 	CloudaccountCreateInput
 
+	// description: Recommended network subnet mask
+	// example: 24
+	NetworkBits *int8 `json:"network_bits"`
+	// description: The upper limit of host number
+	// example: 254
+	HostNumberUpperLimit *int `json:"host_number_upper_limit"`
+	// description: The lower limit of host number
+	// example: 2
+	HostNumberLowerLimit *int `json:"host_number_lower_limit"`
 	// enum: vcenter,datacenter,cluster
 	WireLevelForVmware string `json:"wire_level_for_vmware"`
 }
@@ -427,3 +436,9 @@ type GetCloudaccountSamlOutput struct {
 	// initial SAML SSO login URL for this cloudaccount
 	InitLoginUrl string `json:"init_login_url,allowempty"`
 }
+
+type CloudaccountEnableAutoSyncInput struct {
+	// 云账号状态必须是connected
+	// 最小值为region服务的minimal_sync_interval_seconds
+	SyncIntervalSeconds int `json:"sync_interval_seconds"`
+}

pkg/apis/compute/cloudaccount_const.go

@@ -101,6 +101,50 @@ var (
 		CLOUD_PROVIDER_GOOGLE,
 		CLOUD_PROVIDER_CTYUN,
 	}
+
+	CLOUD_PROVIDER_HOST_TYPE_MAP = map[string][]string{
+		CLOUD_PROVIDER_ONECLOUD: []string{
+			HOST_TYPE_KVM,
+			HOST_TYPE_BAREMETAL,
+			HOST_TYPE_HYPERVISOR,
+		},
+		CLOUD_PROVIDER_VMWARE: []string{
+			HOST_TYPE_ESXI,
+		},
+		CLOUD_PROVIDER_ALIYUN: []string{
+			HOST_TYPE_ALIYUN,
+		},
+		CLOUD_PROVIDER_APSARA: []string{
+			HOST_TYPE_APSARA,
+		},
+		CLOUD_PROVIDER_QCLOUD: []string{
+			HOST_TYPE_QCLOUD,
+		},
+		CLOUD_PROVIDER_AZURE: []string{
+			HOST_TYPE_AZURE,
+		},
+		CLOUD_PROVIDER_AWS: []string{
+			HOST_TYPE_AWS,
+		},
+		CLOUD_PROVIDER_HUAWEI: []string{
+			HOST_TYPE_HUAWEI,
+		},
+		CLOUD_PROVIDER_OPENSTACK: []string{
+			HOST_TYPE_OPENSTACK,
+		},
+		CLOUD_PROVIDER_UCLOUD: []string{
+			HOST_TYPE_UCLOUD,
+		},
+		CLOUD_PROVIDER_ZSTACK: []string{
+			HOST_TYPE_ZSTACK,
+		},
+		CLOUD_PROVIDER_GOOGLE: []string{
+			HOST_TYPE_GOOGLE,
+		},
+		CLOUD_PROVIDER_CTYUN: []string{
+			HOST_TYPE_CTYUN,
+		},
+	}
 )
 
 const (

pkg/apis/compute/cloudregions_const.go

@@ -177,6 +177,9 @@ const (
 	// 尼日利亚
 	CITY_LAGOS = "Lagos" //拉哥斯
 
+	// 巴林王国 (中东国家)
+	CITY_BAHRAIN = "Bahrain" // 巴林
+
 	// 越南
 	CITY_HO_CHI_MINH = "Ho Chi Minh" //???
 
@@ -207,4 +210,5 @@ const (
 	COUNTRY_CODE_NO = "NO" //挪威
 	COUNTRY_CODE_MX = "MX" //墨西哥
 	COUNTRY_CODE_CL = "CL" //智利
+	COUNTRY_CODE_BH = "BH" //巴林
 )

pkg/apis/compute/dbinstance.go

@@ -166,11 +166,7 @@ type SDBInstanceChangeConfigInput struct {
 	apis.Meta
 
 	InstanceType string
-	VCpuCount    int
-	VmemSizeMb   int
-	StorageType  string
 	DiskSizeGB   int
-	Category     string
 }
 
 type SDBInstanceRecoveryConfigInput struct {

pkg/apis/compute/disk.go

@@ -106,6 +106,7 @@ type DiskListInput struct {
 	apis.VirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 	apis.MultiArchResourceBaseListInput
+	apis.AutoDeleteResourceBaseListInput
 	billing.BillingResourceListInput
 	StorageFilterListInput
 
@@ -134,8 +135,6 @@ type DiskListInput struct {
 
 	DiskSize int `json:"disk_size"`
 
-	AutoDelete *bool `json:"auto_delete"`
-
 	FsFormat string `json:"fs_format"`
 
 	// 镜像

pkg/apis/compute/elasticcache.go

@@ -159,6 +159,12 @@ type ElasticcacheCreateInput struct {
 	// 主可用区名称或Id
 	Zone string `json:"zone"`
 
+	// 备可用区名称或Id列表
+	// 默认副本与主可用区一致
+	// 支持此参数的云厂商: 腾讯云
+	// required: false
+	SlaveZones []string `json:"slave_zones"`
+
 	// Ip子网名称或Id,建议使用Id
 	// required: true
 	Network string `json:"network"`

pkg/apis/compute/guests.go

@@ -460,3 +460,25 @@ type ServerStopInput struct {
 	// 目前仅阿里云，腾讯云此参数生效
 	StopCharging bool `json:"stop_charging"`
 }
+
+type ServerDeleteInput struct {
+	// 是否越过回收站直接删除
+	// default: false
+	OverridePendingDelete bool
+
+	// 是否仅删除本地资源
+	// default: false
+	Purge bool
+
+	// 是否删除快照
+	// default: false
+	DeleteSnapshots bool
+
+	// 是否删除关联的EIP
+	// default: false
+	DeleteEip bool
+
+	// 是否删除关联的数据盘
+	// default: false
+	DeleteDisks bool
+}

pkg/apis/compute/schedtag.go

@@ -59,6 +59,7 @@ type SchedtagFilterListInput struct {
 type SchedtagListInput struct {
 	apis.StandaloneResourceListInput
 	apis.ScopedResourceBaseListInput
+	CloudproviderResourceInput
 
 	// fitler by resource_type
 	ResourceType []string `json:"resource_type"`

pkg/apis/list.go

@@ -319,3 +319,7 @@ type EnabledStatusInfrasResourceBaseListInput struct {
 type MultiArchResourceBaseListInput struct {
 	OsArch string `json:"os_arch"`
 }
+
+type AutoDeleteResourceBaseListInput struct {
+	AutoDelete *bool
+}

pkg/apis/logger/input.go

@@ -27,4 +27,14 @@ type BaremetalEventListInput struct {
 	Since time.Time `json:"since"`
 	// until
 	Until time.Time `json:"until"`
+	// host_id
+	HostId []string `json:"host_id"`
+	// id
+	Id []int64 `json:"id"`
+	// EventId
+	EventId []string `json:"event_id"`
+	// Type
+	Type []string `json:"type"`
+	// ipmi_ip
+	IpmiIp []string `json:"ipmi_ip"`
 }

pkg/apis/monitor/alertquery_const.go

@@ -0,0 +1,147 @@
+package monitor
+
+var (
+	ServerTags = map[string]string{
+		"host":             "host",
+		"host_id":          "host_id",
+		"vm_id":            "id",
+		"vm_ip":            "ips",
+		"vm_name":          "name",
+		"zone":             "zone",
+		"zone_id":          "zone_id",
+		"zone_ext_id":      "zone_ext_id",
+		"os_type":          "os_type",
+		"status":           "status",
+		"cloudregion":      "cloudregion",
+		"cloudregion_id":   "cloudregion_id",
+		"region_ext_id":    "region_ext_id",
+		"tenant":           "tenant",
+		"tenant_id":        "tenant_id",
+		"brand":            "brand",
+		"scaling_group_id": "vm_scaling_group_id",
+		"domain_id":        "domain_id",
+		"project_domain":   "project_domain",
+	}
+
+	HostTags = map[string]string{
+		"host_id":        "id",
+		"host_ip":        "ips",
+		"host":           "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	RdsTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"rds_id":         "id",
+		"rds_ip":         "ips",
+		"rds_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	RedisTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"redis_id":       "id",
+		"redis_ip":       "ips",
+		"redis_name":     "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	OssTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"oss_id":         "id",
+		"oss_ip":         "ips",
+		"oss_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	ElbTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"elb_id":         "id",
+		"elb_ip":         "ips",
+		"elb_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"region":         "region",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	CloudAccountTags = map[string]string{
+		"cloudaccount_id":   "id",
+		"cloudaccount_name": "name",
+		"brand":             "brand",
+		"domain_id":         "domain_id",
+		"project_domain":    "project_domain",
+	}
+
+	TenantTags = map[string]string{
+		"tenant_id":      "id",
+		"tenant":         "name",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+	DomainTags = map[string]string{
+		"domain_id":      "id",
+		"project_domain": "name",
+	}
+)

pkg/apis/monitor/metric.go

@@ -9,6 +9,8 @@ const (
 	METRIC_RES_TYPE_OSS          = "oss"
 	METRIC_RES_TYPE_RDS          = "rds"
 	METRIC_RES_TYPE_CLOUDACCOUNT = "cloudaccount"
+	METRIC_RES_TYPE_TENANT       = "tenant"
+	METRIC_RES_TYPE_DOMAIN       = "domain"
 
 	METRIC_UNIT_PERCENT = "%"
 	METRIC_UNIT_BPS     = "bps"

pkg/apis/monitor/unifiedmonitor_const.go

@@ -21,12 +21,14 @@ var (
 		"cloudaccount": "cloudaccount_name",
 	}
 	MEASUREMENT_TAG_ID = map[string]string{
-		"host":         "host_id",
-		"guest":        "vm_id",
-		"redis":        "redis_id",
-		"rds":          "rds_id",
-		"oss":          "oss_id",
-		"cloudaccount": "cloudaccount_id",
+		METRIC_RES_TYPE_HOST:         "host_id",
+		METRIC_RES_TYPE_GUEST:        "vm_id",
+		METRIC_RES_TYPE_REDIS:        "redis_id",
+		METRIC_RES_TYPE_RDS:          "rds_id",
+		METRIC_RES_TYPE_OSS:          "oss_id",
+		METRIC_RES_TYPE_CLOUDACCOUNT: "cloudaccount_id",
+		METRIC_RES_TYPE_TENANT:       "tenant_id",
+		METRIC_RES_TYPE_DOMAIN:       "domain_id",
 	}
 	AlertReduceFunc = map[string]string{
 		"avg":          "average value",

pkg/baremetal/tasks/stop.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
@@ -47,7 +48,7 @@ func NewBaremetalServerStopTask(
 func (task *SBaremetalServerStopTask) DoStop(ctx context.Context, args interface{}) error {
 	task.SetStage(task.WaitForStop)
 	if err := task.Baremetal.DoPowerShutdown(true); err != nil {
-		return errors.Wrap(err, "Do power shutdown error")
+		log.Errorf("Do power shutdown error: %s", err)
 	}
 	task.startTime = time.Now()
 	ExecuteTask(task, nil)
@@ -73,7 +74,7 @@ func (self *SBaremetalServerStopTask) WaitForStop(ctx context.Context, args inte
 		isSoft = false
 	}
 	if err := self.Baremetal.DoPowerShutdown(isSoft); err != nil {
-		return errors.Wrap(err, "DoPowerShutdown")
+		log.Errorf("DoPowerShutdown soft=%v error: %s", isSoft, err)
 	}
 	time.Sleep(10 * time.Second)
 	ExecuteTask(self, nil)

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -115,6 +115,8 @@ func (dev *MegaRaidPhyDev) parseLine(line string) bool {
 			dev.Status = "jbod"
 		} else if strings.Contains(strings.ToLower(val), "online") {
 			dev.Status = "online"
+		} else if val == "Rebuild" {
+			dev.Status = "rebuild"
 		} else {
 			dev.Status = "offline"
 		}
@@ -624,7 +626,7 @@ func (a StorcliAdaptor) key() string {
 }
 
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.key() != ""
+	return a.Controller >= 0 && a.name != "" && a.sn != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -56,7 +56,7 @@ func (dev *Mpt2SASRaidPhyDev) parseLine(line string) bool {
 	}
 	switch key {
 	case "Drive Type":
-		if strings.HasSuffix("_HDD", val) {
+		if strings.HasSuffix(val, "_HDD") {
 			dev.Rotate = tristate.True
 		} else {
 			dev.Rotate = tristate.False

pkg/cloudcommon/db/auto_delete.go

@@ -0,0 +1,54 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by model-api-gen. DO NOT EDIT.
+
+package db
+
+import (
+	"context"
+
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+// +onecloud:model-api-gen
+type SAutoDeleteResourceBase struct {
+	// 是否跟随资源自动删除
+	// example: false
+	AutoDelete bool `nullable:"false" default:"false" get:"user" update:"user" json:"auto_delete"`
+}
+
+type SAutoDeleteResourceBaseManager struct{}
+
+func (manager *SAutoDeleteResourceBaseManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query apis.AutoDeleteResourceBaseListInput,
+) (*sqlchemy.SQuery, error) {
+	if query.AutoDelete != nil {
+		q = q.Equals("auto_delete", *query.AutoDelete)
+	}
+
+	return q, nil
+}
+
+func (self *SAutoDeleteResourceBase) SetAutoDelete(model IModel, userCred mcclient.TokenCredential, autoDelete bool) error {
+	_, err := Update(model, func() error {
+		self.AutoDelete = autoDelete
+		return nil
+	})
+	return err
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -425,9 +425,6 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 					extraResults[i].Update(jsonDict)
 					jsonDict = extraResults[i]
 				}
-				if len(exportKeys) > 0 {
-					jsonDict = jsonDict.CopyIncludes(exportKeys...)
-				}
 				if len(fieldFilter) > 0 {
 					jsonDict = jsonDict.CopyIncludes(fieldFilter...)
 				}
@@ -791,6 +788,9 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if len(retList) != retCount {
 		totalCnt = len(retList)
 	}
+	if query.Contains("export_keys") {
+		retList = getExportCols(query, retList)
+	}
 	paginate := false
 	if !customizeFilters.IsEmpty() {
 		// query not use Limit and Offset, do manual pagination
@@ -822,6 +822,18 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 	return &retResult
 }
 
+func getExportCols(query jsonutils.JSONObject, retList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	var exportKeys stringutils2.SSortedStrings
+	exportKeyStr, _ := query.GetString("export_keys")
+	exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ","))
+	for i := range retList {
+		if len(exportKeys) > 0 {
+			retList[i] = retList[i].(*jsonutils.JSONDict).CopyIncludes(exportKeys...)
+		}
+	}
+	return retList
+}
+
 func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modulebase.ListResult, error) {
 	userCred := fetchUserCredential(ctx)
 

pkg/cloudcommon/etcd/etcd.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
@@ -45,7 +46,8 @@ type SEtcdClient struct {
 	onKeepaliveFailure func()
 	leaseLiving        bool
 
-	watchers map[string]*SEtcdWatcher
+	watchers   map[string]*SEtcdWatcher
+	watchersMu *sync.Mutex
 }
 
 func defaultOnKeepAliveFailed() {
@@ -115,6 +117,7 @@ func NewEtcdClient(opt *SEtcdOptions, onKeepaliveFailure func()) (*SEtcdClient,
 	etcdClient.leaseTtlTimeout = timeoutSeconds
 
 	etcdClient.watchers = make(map[string]*SEtcdWatcher)
+	etcdClient.watchersMu = &sync.Mutex{}
 
 	etcdClient.namespace = opt.EtcdNamspace
 
@@ -306,8 +309,10 @@ func (cli *SEtcdClient) Watch(
 	onModify TEtcdModifyEventFunc,
 	onDelete TEtcdDeleteEventFunc,
 ) error {
+	cli.watchersMu.Lock()
 	_, ok := cli.watchers[prefix]
 	if ok {
+		cli.watchersMu.Unlock()
 		return errors.Errorf("watch prefix %s already registered", prefix)
 	}
 
@@ -318,6 +323,7 @@ func (cli *SEtcdClient) Watch(
 		watcher: watcher,
 		cancel:  cancel,
 	}
+	cli.watchersMu.Unlock()
 
 	prefix = cli.getKey(prefix)
 
@@ -346,6 +352,8 @@ func (cli *SEtcdClient) Watch(
 }
 
 func (cli *SEtcdClient) Unwatch(prefix string) {
+	cli.watchersMu.Lock()
+	defer cli.watchersMu.Unlock()
 	watcher, ok := cli.watchers[prefix]
 	if ok {
 		log.Debugf("unwatch %s", prefix)

pkg/cloudcommon/notifyclient/events.go

@@ -47,6 +47,7 @@ var (
 	ActionPendingDelete SAction = "pending_delete"
 	ActionRebuildRoot   SAction = "rebuild_root"
 	ActionChangeConfig  SAction = "change_config"
+	ActionChangeIpaddr  SAction = "change_ipaddr"
 )
 
 type SEvent struct {

pkg/cloudcommon/notifyclient/notify.go

@@ -339,7 +339,11 @@ func genMsgViaLang(ctx context.Context, p sNotifyParams) ([]npk.SNotifyMessage,
 }
 
 func intelliNotify(ctx context.Context, p sNotifyParams) {
-	log.Infof("recipientId: %v, contacts: %v, event %s priority %s", p.recipientId, p.contacts, p.event, p.priority)
+	log.Infof("recipientId: %v, contactType: %s, contacts: %v, event %s priority %s", p.recipientId, p.channel, p.contacts, p.event, p.priority)
+	// hack
+	if len(p.recipientId) == 0 && len(p.contacts) == 0 {
+		p.contacts = []string{""}
+	}
 	msgs, err := genMsgViaLang(ctx, p)
 	if err != nil {
 		log.Errorf("unable send notification: %v", err)

pkg/cloudprovider/dbinstance.go

@@ -77,10 +77,7 @@ type SManagedDBInstanceCreateConfig struct {
 
 type SManagedDBInstanceChangeConfig struct {
 	DiskSizeGB   int
-	StorageType  string
 	InstanceType string
-	VcpuCount    int
-	VmemSizeMb   int
 }
 
 type SDBInstanceDatabaseCreateConfig struct {

pkg/cloudprovider/objectstore.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"reflect"
 	"regexp"
 	"sort"
 	"strconv"
@@ -110,12 +111,15 @@ type SBucketCORSRule struct {
 }
 
 type SBucketRefererConf struct {
-	// 白名单域名列表
-	WhiteList []string
-	// 黑名单域名列表
-	BlackList []string
+	// 域名列表
+	DomainList []string
+	// 域名列表
+	// enmu: Black-List, White-List
+	RefererType string
 	// 是否允许空referer 访问
 	AllowEmptyRefer bool
+
+	Enabled bool
 }
 
 type SBucketPolicyStatement struct {
@@ -264,8 +268,7 @@ type ICloudBucket interface {
 	DeletePolicy(id []string) ([]SBucketPolicyStatement, error)
 
 	GetTags() (map[string]string, error)
-	SetTags(tags map[string]string) error
-	DeleteTags() error
+	SetTags(tags map[string]string, replace bool) error
 }
 
 type ICloudObject interface {
@@ -829,32 +832,27 @@ func DeleteBucketCORS(ibucket ICloudBucket, id []string) ([]SBucketCORSRule, err
 	return deletedRules, nil
 }
 
-func SetBucketMetadata(ibucket ICloudBucket, tags map[string]string, replace bool) error {
-	newTags := map[string]string{}
-	if replace {
-		newTags = tags
-	} else {
-		oldTags, err := ibucket.GetTags()
-		if err != nil {
-			return errors.Wrap(err, "b.getTags()")
-		}
-		for k, v := range oldTags {
-			if _, ok := tags[k]; !ok {
-				tags[k] = v
-			}
-		}
-		newTags = tags
-	}
-	if len(newTags) == 0 {
-		err := ibucket.DeleteTags()
-		if err != nil {
-			return errors.Wrap(err, "b.DeleteTags()")
-		}
-	} else {
-		err := ibucket.SetTags(newTags)
-		if err != nil {
-			return errors.Wrapf(err, "b.setTags(%s)", jsonutils.Marshal(newTags).String())
-		}
-	}
-	return nil
+type TagsUpdateInfo struct {
+	OldTags map[string]string
+	NewTags map[string]string
+}
+
+func (t TagsUpdateInfo) IsChanged() bool {
+	return !reflect.DeepEqual(t.OldTags, t.NewTags)
+}
+
+func SetBucketTags(iBucket ICloudBucket, tags map[string]string) (TagsUpdateInfo, error) {
+	ret := TagsUpdateInfo{}
+	old, err := iBucket.GetTags()
+	if err != nil {
+		if errors.Cause(err) == ErrNotImplemented || errors.Cause(err) == ErrNotSupported {
+			return ret, nil
+		}
+		return ret, errors.Wrapf(err, "iBucket.GetTags")
+	}
+	ret.OldTags, ret.NewTags = old, tags
+	if !ret.IsChanged() {
+		return ret, nil
+	}
+	return ret, iBucket.SetTags(tags, true)
 }

pkg/compute/models/buckets.go

@@ -480,6 +480,7 @@ func (bucket *SBucket) PostCreate(
 	query jsonutils.JSONObject,
 	data jsonutils.JSONObject,
 ) {
+	bucket.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
 	pendingUsage := SRegionQuota{Bucket: 1}
 	keys, err := bucket.GetQuotaKeys()
 	if err != nil {
@@ -495,10 +496,10 @@ func (bucket *SBucket) PostCreate(
 	bucket.SetStatus(userCred, api.BUCKET_STATUS_START_CREATE, "PostCreate")
 	task, err := taskman.TaskManager.NewTask(ctx, "BucketCreateTask", bucket, userCred, nil, "", "", nil)
 	if err != nil {
-		log.Errorf("BucketCreateTask newTask error %s", err)
-	} else {
-		task.ScheduleRun(nil)
+		bucket.SetStatus(userCred, api.BUCKET_STATUS_CREATE_FAIL, errors.Wrapf(err, "NewTask").Error())
+		return
 	}
+	task.ScheduleRun(nil)
 }
 
 func (bucket *SBucket) ValidateUpdateData(
@@ -538,15 +539,17 @@ func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.Token
 	if err != nil {
 		return errors.Wrap(err, "db.SetExternalId")
 	}
+	tags, _ := bucket.GetAllUserMetadata()
+	if len(tags) > 0 {
+		_, err = cloudprovider.SetBucketTags(extBucket, tags)
+		if err != nil {
+			logclient.AddSimpleActionLog(bucket, logclient.ACT_UPDATE_TAGS, err, userCred, false)
+		}
+	}
 	err = bucket.syncWithCloudBucket(ctx, userCred, extBucket, nil, false)
 	if err != nil {
 		return errors.Wrap(err, "bucket.syncWithCloudBucket")
 	}
-	tags, _ := bucket.GetAllUserMetadata()
-	err = cloudprovider.SetBucketMetadata(extBucket, tags, false)
-	if err != nil {
-		log.Errorf("iBucket.SetMetadata failed: %s", err)
-	}
 	return nil
 }
 
@@ -1518,8 +1521,11 @@ func (bucket *SBucket) PerformSetReferer(
 	if err != nil {
 		return nil, errors.Wrap(err, "GetIBucket")
 	}
+
 	conf := cloudprovider.SBucketRefererConf{
-		WhiteList:       input.WhiteList,
+		Enabled:         input.Enabled,
+		DomainList:      input.DomainList,
+		RefererType:     input.RefererType,
 		AllowEmptyRefer: input.AllowEmptyRefer,
 	}
 
@@ -1554,9 +1560,12 @@ func (bucket *SBucket) GetDetailsReferer(
 	if err != nil {
 		return conf, httperrors.NewInternalServerError("iBucket.GetRefer error %s", err)
 	}
-	conf.WhiteList = referConf.WhiteList
-	conf.BlackList = referConf.BlackList
-	conf.AllowEmptyRefer = referConf.AllowEmptyRefer
+	conf.Enabled = referConf.Enabled
+	if conf.Enabled {
+		conf.DomainList = referConf.DomainList
+		conf.RefererType = referConf.RefererType
+		conf.AllowEmptyRefer = referConf.AllowEmptyRefer
+	}
 
 	return conf, nil
 }
@@ -1888,15 +1897,21 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
 func (bucket *SBucket) OnMetadataUpdated(ctx context.Context, userCred mcclient.TokenCredential) {
 	iBucket, err := bucket.GetIBucket()
 	if err != nil {
-		log.Errorf("bucket.GetIBucket() failed: %s", err)
 		return
 	}
-	tags, _ := bucket.GetAllUserMetadata()
-	err = cloudprovider.SetBucketMetadata(iBucket, tags, false)
+	tags, err := bucket.GetAllUserMetadata()
 	if err != nil {
-		log.Errorf("iBucket.SetMetadata failed: %s", err)
+		return
 	}
-	db.OpsLog.LogEvent(bucket, db.ACT_UPDATE_TAGS, tags, userCred)
+	diff, err := cloudprovider.SetBucketTags(iBucket, tags)
+	if err != nil {
+		logclient.AddSimpleActionLog(bucket, logclient.ACT_UPDATE_TAGS, err, userCred, false)
+		return
+	}
+	if diff.IsChanged() {
+		logclient.AddSimpleActionLog(bucket, logclient.ACT_UPDATE_TAGS, diff, userCred, true)
+	}
+	syncVirtualResourceMetadata(ctx, userCred, bucket, iBucket)
 }
 
 func (manager *SBucketManager) ListItemExportKeys(ctx context.Context,

pkg/compute/models/cachedimages.go

@@ -90,7 +90,7 @@ type SCachedimage struct {
 
 	// 镜像类型, system: 公有云镜像, customized: 自定义镜像
 	// example: system
-	ImageType string `width:"16" default:"customized" list:"user"`
+	ImageType string `width:"16" default:"customized" list:"user" index:"true"`
 }
 
 func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error {

pkg/compute/models/cloudaccounts.go

@@ -434,13 +434,41 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien
 // Performpreparenets searches for suitable network facilities for physical and virtual machines under the cloud account or provides configuration recommendations for network facilities before importing a cloud account.
 func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.CloudaccountPerformPrepareNetsInput) (api.CloudaccountPerformPrepareNetsOutput, error) {
 	var (
-		err    error
-		output api.CloudaccountPerformPrepareNetsOutput
+		err                  error
+		output               api.CloudaccountPerformPrepareNetsOutput
+		networkBits          int8
+		hostNumberUpperLimit int
+		hostNumberLowerLimit int
 	)
 	if input.Provider != api.CLOUD_PROVIDER_VMWARE {
 		return output, httperrors.NewNotSupportedError("not support for cloudaccount with provider '%s'", input.Provider)
 	}
 	// validate first
+	if input.NetworkBits == nil {
+		networkBits = 24
+	} else if *input.NetworkBits <= 0 || *input.NetworkBits >= 32 {
+		return output, httperrors.NewInputParameterError("invalid network_bits, it should be between 0 and 32")
+	} else {
+		networkBits = *input.NetworkBits
+	}
+	maxHostNumber := int((1 << uint(32-networkBits)) - 2)
+	if input.HostNumberUpperLimit == nil {
+		hostNumberUpperLimit = maxHostNumber
+	} else if *input.HostNumberUpperLimit > maxHostNumber {
+		return output, httperrors.NewInputParameterError("invalid host numbers, according to networkBits, it should be between 0 and %d", 2^(32-networkBits)-2)
+	} else {
+		hostNumberUpperLimit = *input.HostNumberUpperLimit
+	}
+
+	minHostNumber := 1
+	if input.HostNumberLowerLimit == nil {
+		hostNumberLowerLimit = minHostNumber
+	} else if *input.HostNumberLowerLimit < minHostNumber || *input.HostNumberLowerLimit > hostNumberUpperLimit {
+		return output, httperrors.NewInputParameterError("invalid host numbers, it should be between %d and %d", minHostNumber, hostNumberUpperLimit)
+	} else {
+		hostNumberLowerLimit = *input.HostNumberLowerLimit
+	}
+
 	ownerId, err := scm.FetchOwnerId(ctx, jsonutils.Marshal(input))
 	if err != nil {
 		return output, errors.Wrap(err, "FetchOwnerId in PerformPrepareNets")
@@ -541,20 +569,26 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre
 		networks[i] = nets
 	}
 	return scm.parseAndSuggest(sParseAndSuggest{
-		NInfos:      nInfos,
-		AccountName: input.Name,
-		ZoneIds:     zoneids,
-		Wires:       wires,
-		Networks:    networks,
+		NInfos:               nInfos,
+		AccountName:          input.Name,
+		ZoneIds:              zoneids,
+		Wires:                wires,
+		Networks:             networks,
+		NetworkBits:          networkBits,
+		HostNumberUpperLimit: hostNumberUpperLimit,
+		HostNumberLowerLimit: hostNumberLowerLimit,
 	}), nil
 }
 
 type sParseAndSuggest struct {
-	NInfos      []sNetworkInfo
-	AccountName string
-	ZoneIds     []string
-	Wires       []SWire
-	Networks    [][]SNetwork
+	NInfos               []sNetworkInfo
+	AccountName          string
+	ZoneIds              []string
+	Wires                []SWire
+	Networks             [][]SNetwork
+	NetworkBits          int8
+	HostNumberUpperLimit int
+	HostNumberLowerLimit int
 }
 
 func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.CloudaccountPerformPrepareNetsOutput {
@@ -580,7 +614,6 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 				Id:     nets[i].Id,
 				WireId: nets[i].WireId,
 			})
-
 		}
 	}
 	ipPoolLen := 0
@@ -662,7 +695,7 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 		}
 
 		if len(noNetHostIP) > 0 {
-			sConfs := scm.suggestHostNetworks(noNetHostIP)
+			sConfs := scm.suggestHostNetworks(noNetHostIP, params.NetworkBits)
 			confs := make([]api.CANetConf, len(sConfs))
 			for i := range confs {
 				confs[i].CASimpleNetConf = sConfs[i]
@@ -704,9 +737,9 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 				if _, ok := existedNets.Get(ip); ok {
 					continue
 				}
-				net := ip.NetAddr(24)
-				netLimitLow := net + 1
-				netLimitUp := net + 254
+				net := ip.NetAddr(params.NetworkBits)
+				netLimitLow := net + netutils.IPV4Addr(params.HostNumberLowerLimit)
+				netLimitUp := net + netutils.IPV4Addr(params.HostNumberUpperLimit)
 				// find startip
 				startIp := ip - 1
 				for ; startIp >= netLimitLow; startIp-- {
@@ -739,7 +772,7 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 						VlanID:       vlan,
 						GuestIpStart: (startIp + 1).String(),
 						GuestIpEnd:   (endIp - 1).String(),
-						GuestIpMask:  24,
+						GuestIpMask:  params.NetworkBits,
 						GuestGateway: (net + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)).
 							String(),
 					},
@@ -795,7 +828,7 @@ func (manager *SCloudaccountManager) fetchEsxiZoneIds() ([]string, error) {
 // The suggestHostNetworks give the suggest config of network contains the ip in 'ips'.
 // The suggested network mask is 24 and the gateway is x.x.x.1.
 // The suggests network is the smallest network segment that meets the above conditions.
-func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr) []api.CASimpleNetConf {
+func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr, mask int8) []api.CASimpleNetConf {
 	if len(ips) == 0 {
 		return nil
 	}
@@ -803,7 +836,6 @@ func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr
 		return ips[i] < ips[j]
 	})
 	var (
-		mask        int8 = 24
 		lastnetAddr netutils.IPV4Addr
 		consequent  []netutils.IPV4Addr
 		ret         []api.CASimpleNetConf
@@ -2314,7 +2346,7 @@ func (self *SCloudaccount) AllowPerformEnableAutoSync(ctx context.Context, userC
 	return db.IsAdminAllowPerform(userCred, self, "enable-auto-sync")
 }
 
-func (self *SCloudaccount) PerformEnableAutoSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SCloudaccount) PerformEnableAutoSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.CloudaccountEnableAutoSyncInput) (jsonutils.JSONObject, error) {
 	if self.EnableAutoSync {
 		return nil, nil
 	}
@@ -2323,10 +2355,11 @@ func (self *SCloudaccount) PerformEnableAutoSync(ctx context.Context, userCred m
 		return nil, httperrors.NewInvalidStatusError("cannot enable auto sync in status %s", self.Status)
 	}
 
-	syncIntervalSecs := int64(0)
-	syncIntervalSecs, _ = data.Int("sync_interval_seconds")
+	if input.SyncIntervalSeconds < options.Options.MinimalSyncIntervalSeconds {
+		return nil, httperrors.NewOutOfRangeError("sync_interval_seconds should be greater than %d", options.Options.MinimalSyncIntervalSeconds)
+	}
 
-	self.enableAutoSync(ctx, userCred, int(syncIntervalSecs))
+	self.enableAutoSync(ctx, userCred, input.SyncIntervalSeconds)
 
 	return nil, nil
 }
@@ -2444,6 +2477,9 @@ func (account *SCloudaccount) needSync() bool {
 	if time.Now().Sub(account.LastSyncEndAt) > time.Duration(account.getSyncIntervalSeconds())*time.Second {
 		return true
 	}
+	if account.EnableAutoSync && !account.LastAutoSync.IsZero() && time.Now().Sub(account.LastAutoSync) > time.Duration(account.getSyncIntervalSeconds()-1)*time.Second {
+		return true
+	}
 	return false
 }
 
@@ -2468,7 +2504,7 @@ func (manager *SCloudaccountManager) initAllRecords() {
 
 func (self *SCloudaccount) CanSync() bool {
 	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING || self.getSyncStatus2() == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING {
-		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second {
+		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > time.Duration(self.getSyncIntervalSeconds())*time.Second {
 			return true
 		} else {
 			return false

pkg/compute/models/cloudaccounts_test.go

@@ -80,8 +80,11 @@ func structureTestData() (sParseAndSuggest, error) {
 		ZoneIds: []string{
 			"zone_test",
 		},
-		Wires:    []SWire{},
-		Networks: [][]SNetwork{},
+		Wires:                []SWire{},
+		Networks:             [][]SNetwork{},
+		NetworkBits:          24,
+		HostNumberUpperLimit: 254,
+		HostNumberLowerLimit: 1,
 	}, nil
 }
 

pkg/compute/models/cloudproviderregions.go

@@ -315,11 +315,11 @@ func (self *SCloudproviderregion) markEndSync(ctx context.Context, userCred mccl
 	log.Debugf("markEndSync deepSync %v", *deepSync)
 	err := self.markEndSyncInternal(userCred, syncResults, deepSync)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "markEndSyncInternal")
 	}
 	err = self.GetProvider().markEndSyncWithLock(ctx, userCred)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "markEndSyncWithLock")
 	}
 	return nil
 }
@@ -335,8 +335,7 @@ func (self *SCloudproviderregion) markEndSyncInternal(userCred mcclient.TokenCre
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to markEndSyncInternal error: %v", err)
-		return err
+		return errors.Wrapf(err, "db.Update")
 	}
 	return nil
 }
@@ -373,11 +372,18 @@ func (set SSyncResultSet) Add(manager db.IModelManager, result compare.SyncResul
 func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange) error {
 	syncResults := SSyncResultSet{}
 
-	self.markSyncing(userCred)
-	defer self.markEndSync(ctx, userCred, syncResults, &syncRange.DeepSync)
-
 	localRegion := self.GetRegion()
 	provider := self.GetProvider()
+
+	self.markSyncing(userCred)
+
+	defer func() {
+		err := self.markEndSync(ctx, userCred, syncResults, &syncRange.DeepSync)
+		if err != nil {
+			log.Errorf("markEndSync for %s(%s) : %v", localRegion.Name, provider.Name, err)
+		}
+	}()
+
 	driver, err := provider.GetProvider()
 	if err != nil {
 		log.Errorf("Failed to get driver, connection problem?")
@@ -480,7 +486,7 @@ func (cpr *SCloudproviderregion) needAutoSyncInternal() bool {
 		region := cpr.GetRegion()
 		log.Debugf("empty region %s! no need to check so frequently", region.GetName())
 	}
-	if time.Now().Sub(cpr.LastSyncEndAt) > time.Duration(intval)*time.Second {
+	if time.Now().Sub(cpr.LastSync) > time.Duration(intval)*time.Second {
 		return true
 	}
 	return false

pkg/compute/models/cloudproviders.go

@@ -181,11 +181,11 @@ func (manager *SCloudproviderManager) GetProviderFieldQuery(field string, isPubl
 	} else if isOnPremise.IsFalse() {
 		q = q.Filter(sqlchemy.IsFalse(account.Field("is_on_premise")))
 	}
-	if len(providers) > 0 {
-		q = q.Filter(sqlchemy.In(account.Field("provider"), providers))
-	}
-	if len(brands) > 0 {
-		q = q.Filter(sqlchemy.In(account.Field("brand"), brands))
+	if len(providers) > 0 || len(brands) > 0 {
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.In(account.Field("provider"), providers),
+			sqlchemy.In(account.Field("brand"), brands),
+		))
 	}
 	return q.SubQuery()
 }
@@ -769,8 +769,7 @@ func (self *SCloudprovider) markEndSync(userCred mcclient.TokenCredential) error
 		return nil
 	})
 	if err != nil {
-		log.Errorf("Failed to markEndSync error: %v", err)
-		return err
+		return errors.Wrapf(err, "markEndSync")
 	}
 	return nil
 }
@@ -1347,7 +1346,10 @@ func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, us
 		}
 	}
 	if syncCnt == 0 {
-		provider.markEndSyncWithLock(ctx, userCred)
+		err := provider.markEndSyncWithLock(ctx, userCred)
+		if err != nil {
+			log.Errorf("markEndSyncWithLock for %s error: %v", provider.Name, err)
+		}
 	}
 }
 
@@ -1497,27 +1499,53 @@ func (self *SCloudprovider) PerformDisable(ctx context.Context, userCred mcclien
 	return nil, nil
 }
 
+func (manager *SCloudproviderManager) filterByDomainId(q *sqlchemy.SQuery, domainId string) *sqlchemy.SQuery {
+	subq := db.SharedResourceManager.Query("resource_id")
+	subq = subq.Equals("resource_type", CloudaccountManager.Keyword())
+	subq = subq.Equals("target_project_id", domainId)
+	subq = subq.Equals("target_type", db.SharedTargetDomain)
+
+	cloudaccounts := CloudaccountManager.Query().SubQuery()
+	q = q.Join(cloudaccounts, sqlchemy.Equals(
+		q.Field("cloudaccount_id"),
+		cloudaccounts.Field("id"),
+	))
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.AND(
+			sqlchemy.Equals(q.Field("domain_id"), domainId),
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
+		),
+		sqlchemy.AND(
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
+			sqlchemy.OR(
+				sqlchemy.AND(
+					sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeNone),
+					sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+				),
+				sqlchemy.AND(
+					sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeDomain),
+					sqlchemy.OR(
+						sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+						sqlchemy.In(cloudaccounts.Field("id"), subq.SubQuery()),
+					),
+				),
+				sqlchemy.Equals(cloudaccounts.Field("public_scope"), rbacutils.ScopeSystem),
+			),
+		),
+		sqlchemy.AND(
+			sqlchemy.Equals(cloudaccounts.Field("domain_id"), domainId),
+			sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
+		),
+	))
+	return q
+}
+
 func (manager *SCloudproviderManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
 	if owner != nil {
 		switch scope {
 		case rbacutils.ScopeProject, rbacutils.ScopeDomain:
 			if len(owner.GetProjectDomainId()) > 0 {
-				cloudaccounts := CloudaccountManager.Query().SubQuery()
-				q = q.Join(cloudaccounts, sqlchemy.Equals(
-					q.Field("cloudaccount_id"),
-					cloudaccounts.Field("id"),
-				))
-				q = q.Filter(sqlchemy.OR(
-					sqlchemy.AND(
-						sqlchemy.Equals(q.Field("domain_id"), owner.GetProjectDomainId()),
-						sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
-					),
-					sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
-					sqlchemy.AND(
-						sqlchemy.Equals(cloudaccounts.Field("domain_id"), owner.GetProjectDomainId()),
-						sqlchemy.Equals(cloudaccounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
-					),
-				))
+				q = manager.filterByDomainId(q, owner.GetProjectDomainId())
 			}
 		}
 	}

pkg/compute/models/cloudregions.go

@@ -628,28 +628,17 @@ func (manager *SCloudregionManager) InitializeData() error {
 }
 
 func getCloudRegionIdByDomainId(domainId string) *sqlchemy.SSubQuery {
-	accounts := CloudaccountManager.Query().SubQuery()
 	cloudproviderregions := CloudproviderRegionManager.Query().SubQuery()
-	providers := CloudproviderManager.Query().SubQuery()
 
 	// not managed region
 	q1 := CloudregionManager.Query("id").Equals("provider", api.CLOUD_PROVIDER_ONECLOUD)
 
 	// managed region
 	q2 := cloudproviderregions.Query(cloudproviderregions.Field("cloudregion_id", "id"))
-	q2 = q2.Join(providers, sqlchemy.Equals(providers.Field("id"), cloudproviderregions.Field("cloudprovider_id")))
-	q2 = q2.Join(accounts, sqlchemy.Equals(providers.Field("cloudaccount_id"), accounts.Field("id")))
-	q2 = q2.Filter(sqlchemy.OR(
-		sqlchemy.AND(
-			sqlchemy.Equals(providers.Field("domain_id"), domainId),
-			sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_PROVIDER_DOMAIN),
-		),
-		sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_SYSTEM),
-		sqlchemy.AND(
-			sqlchemy.Equals(accounts.Field("domain_id"), domainId),
-			sqlchemy.Equals(accounts.Field("share_mode"), api.CLOUD_ACCOUNT_SHARE_MODE_ACCOUNT_DOMAIN),
-		),
-	))
+	providerIds := CloudproviderManager.Query("id")
+	providerIds = CloudproviderManager.filterByDomainId(providerIds, domainId)
+	providersIdsQ := providerIds.Distinct().SubQuery()
+	q2 = q2.Join(providersIdsQ, sqlchemy.Equals(providersIdsQ.Field("id"), cloudproviderregions.Field("cloudprovider_id")))
 
 	return sqlchemy.Union(q1, q2).Query().SubQuery()
 }

pkg/compute/models/cloudsync.go

@@ -45,7 +45,7 @@ type SSyncableBaseResourceManager struct{}
 
 func (self *SSyncableBaseResource) CanSync() bool {
 	if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING {
-		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second {
+		if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > time.Duration(options.Options.MinimalSyncIntervalSeconds)*time.Second {
 			return true
 		} else {
 			return false

pkg/compute/models/dbinstance_skus.go

@@ -298,6 +298,11 @@ func (manager *SDBInstanceSkuManager) GetPropertyInstanceSpecs(ctx context.Conte
 		return nil, errors.Wrap(err, "manager.ListItemFilter")
 	}
 
+	q2, err := manager.ListItemFilter(ctx, manager.Query(), userCred, listQuery)
+	if err != nil {
+		return nil, errors.Wrap(err, "manager.ListItemFilter")
+	}
+
 	input := &SDBInstanceSku{}
 	query.Unmarshal(input)
 	for k, v := range map[string]interface{}{
@@ -317,15 +322,30 @@ func (manager *SDBInstanceSkuManager) GetPropertyInstanceSpecs(ctx context.Conte
 			value := v.(string)
 			if len(value) > 0 {
 				q = q.Equals(k, v)
+				q2 = q2.Equals(k, v)
 			}
 		case int, int64:
 			value := fmt.Sprintf("%d", v)
 			if value != "0" {
 				q = q.Equals(k, v)
+				q2 = q2.Equals(k, v)
 			}
 		}
 	}
 
+	sq := q2.SubQuery()
+	q2 = sq.Query(sq.Field("zone1"), sq.Field("zone2"), sq.Field("zone3")).Distinct()
+
+	skuZones := []struct {
+		Zone1 string
+		Zone2 string
+		Zone3 string
+	}{}
+	err = q2.All(&skuZones)
+	if err != nil {
+		return nil, errors.Wrapf(err, "query sku zones")
+	}
+
 	skus := []SDBInstanceSku{}
 	q = q.GroupBy(q.Field("vcpu_count"), q.Field("vmem_size_mb"))
 	q = q.Asc(q.Field("vcpu_count"), q.Field("vmem_size_mb"))
@@ -360,23 +380,7 @@ func (manager *SDBInstanceSkuManager) GetPropertyInstanceSpecs(ctx context.Conte
 	result.Zones.Zone1 = map[string]string{}
 	result.Zones.Zone2 = map[string]string{}
 	result.Zones.Zone3 = map[string]string{}
-	for _, sku := range skus {
-		if _, ok := result.cpuMemsMb[sku.VcpuCount]; !ok {
-			result.cpuMemsMb[sku.VcpuCount] = map[int]bool{}
-			result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)] = []int{}
-			result.Cpus = append(result.Cpus, sku.VcpuCount)
-		}
-
-		if _, ok := result.cpuMemsMb[sku.VcpuCount][sku.VmemSizeMb]; !ok {
-			result.cpuMemsMb[sku.VcpuCount][sku.VmemSizeMb] = true
-			result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)] = append(result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)], sku.VmemSizeMb)
-		}
-
-		if _, ok := result.memsMb[sku.VmemSizeMb]; !ok {
-			result.memsMb[sku.VmemSizeMb] = true
-			result.MemsMb = append(result.MemsMb, sku.VmemSizeMb)
-		}
-
+	for _, sku := range skuZones {
 		if _, ok := result.Zones.Zone1[sku.Zone1]; !ok && len(sku.Zone1) > 0 {
 			result.Zones.Zone1[sku.Zone1] = sku.Zone1
 		}
@@ -403,20 +407,36 @@ func (manager *SDBInstanceSkuManager) GetPropertyInstanceSpecs(ctx context.Conte
 			result.Zones.Zones[zoneId] = zoneId
 		}
 	}
-
-	zq := ZoneManager.Query("id", "name").In("id", result.zones)
-	rows, err := zq.Rows()
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	for rows.Next() {
-		id, name := "", ""
-		err = rows.Scan(&id, &name)
-		if err != nil {
-			return nil, errors.Wrap(err, "rows.Scan")
+	for _, sku := range skus {
+		if _, ok := result.cpuMemsMb[sku.VcpuCount]; !ok {
+			result.cpuMemsMb[sku.VcpuCount] = map[int]bool{}
+			result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)] = []int{}
+			result.Cpus = append(result.Cpus, sku.VcpuCount)
 		}
-		result.zoneNames[id] = name
+
+		if _, ok := result.cpuMemsMb[sku.VcpuCount][sku.VmemSizeMb]; !ok {
+			result.cpuMemsMb[sku.VcpuCount][sku.VmemSizeMb] = true
+			result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)] = append(result.CpuMemsMb[fmt.Sprintf("%d", sku.VcpuCount)], sku.VmemSizeMb)
+		}
+
+		if _, ok := result.memsMb[sku.VmemSizeMb]; !ok {
+			result.memsMb[sku.VmemSizeMb] = true
+			result.MemsMb = append(result.MemsMb, sku.VmemSizeMb)
+		}
+
+	}
+
+	zones := []struct {
+		Id   string
+		Name string
+	}{}
+
+	err = ZoneManager.Query("id", "name").In("id", result.zones).All(&zones)
+	if err != nil {
+		return nil, errors.Wrapf(err, "query zones")
+	}
+	for _, zone := range zones {
+		result.zoneNames[zone.Id] = zone.Name
 	}
 
 	for _, zoneId := range result.Zones.Zone1 {

pkg/compute/models/dbinstances.go

@@ -420,7 +420,7 @@ func (man *SDBInstanceManager) ValidateCreateData(ctx context.Context, userCred
 
 	instance := SDBInstance{}
 	jsonutils.Update(&instance, input)
-	skus, err := instance.GetAvailableDBInstanceSkus()
+	skus, err := instance.GetAvailableDBInstanceSkus(false)
 	if err != nil {
 		return input, httperrors.NewGeneralError(err)
 	}
@@ -1062,57 +1062,20 @@ func (self *SDBInstance) AllowPerformChangeConfig(ctx context.Context, userCred
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "change-config")
 }
 
-func (self *SDBInstance) PerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SDBInstance) PerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SDBInstanceChangeConfigInput) (jsonutils.JSONObject, error) {
 	if !utils.IsInStringArray(self.Status, []string{api.DBINSTANCE_RUNNING}) {
 		return nil, httperrors.NewInputParameterError("Cannot change config in status %s", self.Status)
 	}
-	input := api.SDBInstanceChangeConfigInput{}
-	err := data.Unmarshal(&input)
-	if err != nil {
-		return nil, httperrors.NewInputParameterError("Unmarshal input error: %v", err)
+
+	if input.DiskSizeGB != 0 && input.DiskSizeGB < self.DiskSizeGB {
+		return nil, httperrors.NewUnsupportOperationError("DBInstance Disk cannot be thrink")
 	}
 
-	tmp := &SDBInstance{}
-	jsonutils.Update(tmp, self)
-
-	if len(input.StorageType) > 0 {
-		self.StorageType = input.StorageType
+	if input.DiskSizeGB == self.DiskSizeGB && input.InstanceType == self.InstanceType {
+		return nil, nil
 	}
 
-	changed := false
-	if len(input.InstanceType) > 0 {
-		tmp.InstanceType = input.InstanceType
-		changed = true
-	} else if input.VCpuCount > 0 {
-		tmp.VcpuCount = input.VCpuCount
-		self.InstanceType = ""
-		changed = true
-	} else if input.VmemSizeMb > 0 {
-		tmp.VmemSizeMb = input.VmemSizeMb
-		tmp.InstanceType = ""
-		changed = true
-	} else if len(input.Category) > 0 {
-		tmp.Category = input.Category
-		tmp.InstanceType = ""
-		changed = true
-	}
-
-	if changed {
-		skus, err := tmp.GetAvailableDBInstanceSkus()
-		if err != nil {
-			return nil, httperrors.NewGeneralError(errors.Wrap(err, "self.GetAvailableDBInstanceSkus"))
-		}
-		if len(skus) == 0 {
-			return nil, httperrors.NewInputParameterError("failed to match any skus for change config")
-		}
-	}
-
-	err = self.GetRegion().GetDriver().ValidateChangeDBInstanceConfigData(ctx, userCred, self, &input)
-	if err != nil {
-		return nil, err
-	}
-
-	return nil, self.StartDBInstanceChangeConfig(ctx, userCred, data.(*jsonutils.JSONDict), "")
+	return nil, self.StartDBInstanceChangeConfig(ctx, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), "")
 }
 
 func (self *SDBInstance) StartDBInstanceChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error {
@@ -1511,14 +1474,18 @@ func (self *SDBInstance) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
-func (self *SDBInstance) GetDBInstanceSkuQuery() *sqlchemy.SQuery {
+func (self *SDBInstance) GetDBInstanceSkuQuery(skipZoneCheck bool) *sqlchemy.SQuery {
 	q := DBInstanceSkuManager.Query().Equals("storage_type", self.StorageType).Equals("category", self.Category).
 		Equals("cloudregion_id", self.CloudregionId).Equals("engine", self.Engine).Equals("engine_version", self.EngineVersion)
-	for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3} {
-		if len(v) > 0 {
-			q = q.Equals(k, v)
+
+	if !skipZoneCheck {
+		for k, v := range map[string]string{"zone1": self.Zone1, "zone2": self.Zone2, "zone3": self.Zone3} {
+			if len(v) > 0 {
+				q = q.Equals(k, v)
+			}
 		}
 	}
+
 	if len(self.InstanceType) > 0 {
 		q = q.Equals("name", self.InstanceType)
 	} else {
@@ -1527,9 +1494,9 @@ func (self *SDBInstance) GetDBInstanceSkuQuery() *sqlchemy.SQuery {
 	return q
 }
 
-func (self *SDBInstance) GetAvailableDBInstanceSkus() ([]SDBInstanceSku, error) {
+func (self *SDBInstance) GetAvailableDBInstanceSkus(skipZoneCheck bool) ([]SDBInstanceSku, error) {
 	skus := []SDBInstanceSku{}
-	q := self.GetDBInstanceSkuQuery().Equals("status", api.DBINSTANCE_SKU_AVAILABLE)
+	q := self.GetDBInstanceSkuQuery(skipZoneCheck).Equals("status", api.DBINSTANCE_SKU_AVAILABLE)
 	err := db.FetchModelObjects(DBInstanceSkuManager, q, &skus)
 	if err != nil {
 		return nil, err
@@ -1538,9 +1505,9 @@ func (self *SDBInstance) GetAvailableDBInstanceSkus() ([]SDBInstanceSku, error)
 
 }
 
-func (self *SDBInstance) GetDBInstanceSkus() ([]SDBInstanceSku, error) {
+func (self *SDBInstance) GetDBInstanceSkus(skipZoneCheck bool) ([]SDBInstanceSku, error) {
 	skus := []SDBInstanceSku{}
-	q := self.GetDBInstanceSkuQuery()
+	q := self.GetDBInstanceSkuQuery(skipZoneCheck)
 	err := db.FetchModelObjects(DBInstanceSkuManager, q, &skus)
 	if err != nil {
 		return nil, err
@@ -1548,23 +1515,9 @@ func (self *SDBInstance) GetDBInstanceSkus() ([]SDBInstanceSku, error) {
 	return skus, nil
 }
 
-func (self *SDBInstance) GetAvailableZoneIds() ([]string, error) {
-	zoneIds := []string{}
-	skus, err := self.GetDBInstanceSkus()
-	if err != nil {
-		return nil, errors.Wrap(err, "self.GetDBInstanceSkus")
-	}
-	for _, sku := range skus {
-		if !utils.IsInStringArray(sku.ZoneId, zoneIds) {
-			zoneIds = append(zoneIds, sku.ZoneId)
-		}
-	}
-	return zoneIds, nil
-}
-
 func (self *SDBInstance) GetAvailableInstanceTypes() ([]cloudprovider.SInstanceType, error) {
 	instanceTypes := []cloudprovider.SInstanceType{}
-	skus, err := self.GetAvailableDBInstanceSkus()
+	skus, err := self.GetAvailableDBInstanceSkus(false)
 	if err != nil {
 		return nil, errors.Wrap(err, "self.GetAvailableDBInstanceSkus")
 	}
@@ -1580,7 +1533,7 @@ func (self *SDBInstance) GetAvailableInstanceTypes() ([]cloudprovider.SInstanceT
 
 func (self *SDBInstance) setZoneInfo() error {
 	sku := SDBInstanceSku{}
-	q := self.GetDBInstanceSkuQuery()
+	q := self.GetDBInstanceSkuQuery(false)
 	count, err := q.CountWithError()
 	if err != nil {
 		return errors.Wrapf(err, "q.CountWithError")

pkg/compute/models/disks.go

@@ -60,6 +60,7 @@ type SDiskManager struct {
 	SStorageResourceBaseManager
 	SBillingResourceBaseManager
 	db.SMultiArchResourceBaseManager
+	db.SAutoDeleteResourceBaseManager
 }
 
 var DiskManager *SDiskManager
@@ -83,6 +84,7 @@ type SDisk struct {
 	SBillingResourceBase
 	SStorageResourceBase `width:"128" charset:"ascii" nullable:"true" list:"admin" create:"optional"`
 	db.SMultiArchResourceBase
+	db.SAutoDeleteResourceBase
 
 	// 磁盘存储类型
 	// example: qcow2
@@ -93,10 +95,6 @@ type SDisk struct {
 	// 磁盘路径
 	AccessPath string `width:"256" charset:"ascii" nullable:"true" get:"user" json:"access_path"`
 
-	// 是否跟随云主机自动删除, 仅绑定到云主机时才生效
-	// example: false
-	AutoDelete bool `nullable:"false" default:"false" get:"user" update:"user" json:"auto_delete"`
-
 	// 存储Id
 	// StorageId       string `width:"128" charset:"ascii" nullable:"true" list:"admin" create:"optional"`
 
@@ -174,6 +172,11 @@ func (manager *SDiskManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SMultiArchResourceBaseManager.ListItemFilter")
 	}
 
+	q, err = manager.SAutoDeleteResourceBaseManager.ListItemFilter(ctx, q, userCred, query.AutoDeleteResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrapf(err, "SAutoDeleteResourceBaseManager.ListItemFilter")
+	}
+
 	if query.Unused != nil {
 		guestdisks := GuestdiskManager.Query().SubQuery()
 		sq := guestdisks.Query(guestdisks.Field("disk_id"))
@@ -224,14 +227,6 @@ func (manager *SDiskManager) ListItemFilter(
 		q = q.Equals("disk_size", query.DiskSize)
 	}
 
-	if query.AutoDelete != nil {
-		if *query.AutoDelete {
-			q = q.IsTrue("auto_delete")
-		} else {
-			q = q.IsFalse("auto_delete")
-		}
-	}
-
 	if len(query.FsFormat) > 0 {
 		q = q.Equals("fs_format", query.FsFormat)
 	}

pkg/compute/models/elasticcache_instances.go

@@ -83,7 +83,7 @@ type SElasticcache struct {
 	SZoneResourceBase
 
 	// 备可用区
-	SlaveZones string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" json:"slave_zones"`
+	SlaveZones string `width:"512" charset:"ascii" nullable:"false" list:"user" create:"optional" json:"slave_zones"`
 
 	// 实例规格
 	// example: redis.master.micro.default
@@ -757,24 +757,28 @@ func (manager *SElasticcacheManager) AllowCreateItem(ctx context.Context, userCr
 }
 
 func (manager *SElasticcacheManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
+	_data := api.ElasticcacheCreateInput{}
+	err := data.Unmarshal(&_data)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "ElasticcacheCreateInput.Unmarshal")
+	}
+	input, err := manager.validateCreateData(ctx, userCred, ownerId, query, _data)
+	if err != nil {
+		return nil, errors.Wrap(err, "validateCreateData")
 	}
 
 	return input, nil
 }
 
 func (manager *SElasticcacheManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
-	data := input.JSON(&input)
-	return manager.validateCreateData(ctx, userCred, ownerId, query, data)
+	return manager.validateCreateData(ctx, userCred, ownerId, query, input)
 }
 
-func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
 	var region *SCloudregion
 	var provider *SCloudprovider
-	if id, _ := data.GetString("network"); len(id) > 0 {
-		network, err := db.FetchByIdOrName(NetworkManager, userCred, strings.Split(id, ",")[0])
+	if len(input.Network) > 0 {
+		network, err := db.FetchByIdOrName(NetworkManager, userCred, strings.Split(input.Network, ",")[0])
 		if err != nil {
 			return nil, fmt.Errorf("getting network failed")
 		}
@@ -788,34 +792,28 @@ func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, use
 	}
 
 	// postpiad billing cycle
-	billingType, _ := data.GetString("billing_type")
-	if billingType == billing_api.BILLING_TYPE_POSTPAID {
-		billingCycle, _ := data.GetString("duration")
-		if len(billingCycle) > 0 {
-			cycle, err := bc.ParseBillingCycle(billingCycle)
+	if input.BillingType == billing_api.BILLING_TYPE_POSTPAID {
+		_cycle := input.Duration
+		if len(_cycle) > 0 {
+			cycle, err := bc.ParseBillingCycle(_cycle)
 			if err != nil {
-				return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", billingCycle)
+				return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", _cycle)
 			}
 
 			tm := time.Time{}
-			data.Set("billing_cycle", jsonutils.NewString(cycle.String()))
-			data.Set("expired_at", jsonutils.NewString(cycle.EndAt(tm).Format("2006-01-02 15:04:05")))
+			input.BillingCycle = cycle.String()
+			// .Format("2006-01-02 15:04:05")
+			input.ExpiredAt = cycle.EndAt(tm)
 		}
 	}
 
-	input := apis.VirtualResourceCreateInput{}
 	var err error
-	err = data.Unmarshal(&input)
+	input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
 	if err != nil {
-		return nil, httperrors.NewInternalServerError("unmarshal VirtualResourceCreateInput fail %s", err)
+		return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ValidateCreateData")
 	}
-	input, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input)
-	if err != nil {
-		return nil, err
-	}
-	data.Update(jsonutils.Marshal(input))
 
-	ret, err := region.GetDriver().ValidateCreateElasticcacheData(ctx, userCred, nil, data)
+	ret, err := region.GetDriver().ValidateCreateElasticcacheData(ctx, userCred, nil, input)
 	if err != nil {
 		return nil, errors.Wrap(err, "region.GetDriver().ValidateCreateElasticcacheData")
 	}
@@ -866,6 +864,36 @@ func (self *SElasticcache) StartElasticcacheCreateTask(ctx context.Context, user
 	return nil
 }
 
+func (self *SElasticcache) GetSlaveZones() ([]SZone, error) {
+	if len(self.SlaveZones) > 0 {
+		zones := []SZone{}
+		sz := strings.Split(self.SlaveZones, ",")
+		err := ZoneManager.Query().In("id", sz).All(&zones)
+		if err != nil {
+			return nil, errors.Wrap(err, "GetZones")
+		}
+
+		zoneMap := map[string]SZone{}
+		for i := range zones {
+			zoneMap[zones[i].GetId()] = zones[i]
+		}
+
+		ret := make([]SZone, len(sz))
+		for i := range sz {
+			z, ok := zoneMap[sz[i]]
+			if !ok {
+				return nil, fmt.Errorf("zone %s is not found", sz[i])
+			}
+
+			ret[i] = z
+		}
+
+		return ret, nil
+	}
+
+	return []SZone{}, nil
+}
+
 /*func (self *SElasticcache) GetIRegion() (cloudprovider.ICloudRegion, error) {
 	provider, err := self.GetDriver()
 	if err != nil {
@@ -1080,11 +1108,21 @@ func (self *SElasticcache) GetCreateQCloudElasticcacheParams(data *jsonutils.JSO
 
 	zone := self.GetZone()
 	if zone != nil {
-		izone, err := iregion.GetIZoneById(zone.ExternalId)
+		zones := []SZone{*zone}
+		// slave zones
+		sz, err := self.GetSlaveZones()
 		if err != nil {
-			return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone")
+			return nil, errors.Wrap(err, "GetSlaveZones")
+		}
+
+		zones = append(zones, sz...)
+		for i := range zones {
+			izone, err := iregion.GetIZoneById(zones[i].ExternalId)
+			if err != nil {
+				return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone")
+			}
+			input.ZoneIds = append(input.ZoneIds, izone.GetId())
 		}
-		input.ZoneIds = []string{izone.GetId()}
 	}
 
 	switch self.BillingType {

pkg/compute/models/guest_actions.go

@@ -1086,24 +1086,17 @@ func (self *SGuest) StartSyncstatus(ctx context.Context, userCred mcclient.Token
 
 func (self *SGuest) StartAutoDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
 	db.OpsLog.LogEvent(self, db.ACT_DELETE, "auto-delete after stop", userCred)
-	return self.StartDeleteGuestTask(ctx, userCred, parentTaskId, false, false, false)
+	opts := api.ServerDeleteInput{}
+	return self.StartDeleteGuestTask(ctx, userCred, parentTaskId, opts)
 }
 
 func (self *SGuest) StartDeleteGuestTask(
 	ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string,
-	isPurge, overridePendingDelete, deleteSnapshots bool,
+	opts api.ServerDeleteInput,
 ) error {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(self.Status), "guest_status")
-	if isPurge {
-		params.Add(jsonutils.JSONTrue, "purge")
-	}
-	if overridePendingDelete {
-		params.Add(jsonutils.JSONTrue, "override_pending_delete")
-	}
-	if deleteSnapshots {
-		params.Add(jsonutils.JSONTrue, "delete_snapshots")
-	}
+	params.Update(jsonutils.Marshal(opts))
 	self.SetStatus(userCred, api.VM_START_DELETE, "")
 	return self.GetDriver().StartDeleteGuestTask(ctx, userCred, self, params, parentTaskId)
 }
@@ -1400,7 +1393,10 @@ func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCre
 	if host != nil && host.GetEnabled() {
 		return nil, httperrors.NewInvalidStatusError("Cannot purge server on enabled host")
 	}
-	err = self.StartDeleteGuestTask(ctx, userCred, "", true, false, false)
+	opts := api.ServerDeleteInput{
+		Purge: true,
+	}
+	err = self.StartDeleteGuestTask(ctx, userCred, "", opts)
 	return nil, err
 }
 
@@ -2131,6 +2127,7 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T
 		notes.Add(jsonutils.NewString(ngn[0].IpAddr), "ip")
 	}
 	logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_CHANGE_NIC, notes, userCred, true)
+	notifyclient.NotifyWebhook(ctx, userCred, self, notifyclient.ActionChangeIpaddr)
 
 	err = self.StartSyncTask(ctx, userCred, true, "")
 	return nil, err
@@ -2219,7 +2216,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		if err != nil {
 			return nil, err
 		}
-		if IsExitNetworkInfo(input.Nets[i]) {
+		if IsExitNetworkInfo(userCred, input.Nets[i]) {
 			enicCnt = count
 			// ebw = input.BwLimit
 		} else {

pkg/compute/models/guests.go

@@ -1747,7 +1747,7 @@ func getGuestResourceRequirements(
 	eBw := 0
 	iBw := 0
 	for _, netConfig := range input.Networks {
-		if IsExitNetworkInfo(netConfig) {
+		if IsExitNetworkInfo(userCred, netConfig) {
 			eNicCnt += 1
 			eBw += netConfig.BwLimit
 		} else {
@@ -2676,8 +2676,9 @@ func (manager *SGuestManager) TotalCount(
 	status []string, hypervisors []string,
 	includeSystem bool, pendingDelete bool,
 	hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string,
+	since *time.Time,
 ) SGuestCountStat {
-	return usageTotalGuestResouceCount(scope, ownerId, rangeObjs, status, hypervisors, includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	return usageTotalGuestResouceCount(scope, ownerId, rangeObjs, status, hypervisors, includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv, since)
 }
 
 func (self *SGuest) detachNetworks(ctx context.Context, userCred mcclient.TokenCredential, gns []SGuestnetwork, reserve bool, deploy bool) error {
@@ -2701,12 +2702,26 @@ func (self *SGuest) getAttach2NetworkCount(net *SNetwork) (int, error) {
 	return q.CountWithError()
 }
 
-func (self *SGuest) getMaxNicIndex() int8 {
+func (self *SGuest) getUsableNicIndex() int8 {
 	nics, err := self.GetNetworks("")
 	if err != nil {
 		return -1
 	}
-	return int8(len(nics))
+	maxIndex := int8(len(nics))
+	for i := int8(0); i <= maxIndex; i++ {
+		found := true
+		for j := range nics {
+			if nics[j].Index == i {
+				found = false
+				break
+			}
+		}
+		if found {
+			return i
+		}
+	}
+	panic(fmt.Sprintf("cannot find usable nic index for guest %s(%s)",
+		self.Name, self.Id))
 }
 
 func (self *SGuest) setOSProfile(ctx context.Context, userCred mcclient.TokenCredential, profile jsonutils.JSONObject) error {
@@ -2847,7 +2862,7 @@ func (self *SGuest) attach2NetworkOnce(
 		nicDriver = args.nicDriver
 	)
 	if index < 0 {
-		index = self.getMaxNicIndex()
+		index = self.getUsableNicIndex()
 	}
 	if nicDriver == "" {
 		osProf := self.GetOSProfile()
@@ -3272,9 +3287,10 @@ func usageTotalGuestResouceCount(
 	hostTypes []string,
 	resourceTypes []string,
 	providers []string, brands []string, cloudEnv string,
+	since *time.Time,
 ) SGuestCountStat {
 	q, guests := _guestResourceCountQuery(scope, ownerId, rangeObjs, status, hypervisors,
-		pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv)
+		pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv, since)
 	if !includeSystem {
 		q = q.Filter(sqlchemy.OR(
 			sqlchemy.IsNull(guests.Field("is_system")), sqlchemy.IsFalse(guests.Field("is_system"))))
@@ -3302,6 +3318,7 @@ func _guestResourceCountQuery(
 	hostTypes []string,
 	resourceTypes []string,
 	providers []string, brands []string, cloudEnv string,
+	since *time.Time,
 ) (*sqlchemy.SQuery, *sqlchemy.SSubQuery) {
 
 	guestdisks := GuestdiskManager.Query().SubQuery()
@@ -3328,7 +3345,12 @@ func _guestResourceCountQuery(
 
 	isoDevSubQuery := isoDevQuery.SubQuery()
 
-	guests := GuestManager.Query().SubQuery()
+	var guests *sqlchemy.SSubQuery
+	if since != nil && !since.IsZero() {
+		guests = GuestManager.RawQuery().SubQuery()
+	} else {
+		guests = GuestManager.Query().SubQuery()
+	}
 	guestBackupSubQuery := GuestManager.Query(
 		"id",
 		"vcpu_count",
@@ -3377,6 +3399,11 @@ func _guestResourceCountQuery(
 	} else {
 		q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(guests.Field("pending_deleted")), sqlchemy.IsFalse(guests.Field("pending_deleted"))))
 	}
+
+	if since != nil && !since.IsZero() {
+		q = q.Filter(sqlchemy.GT(guests.Field("created_at"), *since))
+	}
+
 	return q, guests
 }
 
@@ -3803,18 +3830,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token
 	return self.IsOwner(userCred) || db.IsAdminAllowDelete(userCred, self)
 }
 
-func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	var (
-		overridePendingDelete = false
-		purge                 = false
-		deleteSnapshots       = false
-	)
-	if query != nil {
-		overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false)
-		purge = jsonutils.QueryBoolean(query, "purge", false)
-		deleteSnapshots = jsonutils.QueryBoolean(query, "delete_snapshots", false)
-	}
-	return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete, deleteSnapshots)
+// 删除虚拟机
+func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query api.ServerDeleteInput, data jsonutils.JSONObject) error {
+	return self.StartDeleteGuestTask(ctx, userCred, "", query)
 }
 
 func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.TokenCredential) error {
@@ -4624,7 +4642,10 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use
 		return
 	}
 	for i := 0; i < len(guests); i += 1 {
-		guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true, false)
+		opts := api.ServerDeleteInput{
+			OverridePendingDelete: true,
+		}
+		guests[i].StartDeleteGuestTask(ctx, userCred, "", opts)
 	}
 }
 
@@ -4695,7 +4716,7 @@ func (manager *SGuestManager) DeleteExpiredPrepaidServers(ctx context.Context, u
 	if guests == nil {
 		return
 	}
-	deteleSnapshot := options.Options.DeleteSnapshotExpiredRelease
+	deleteSnapshot := options.Options.DeleteSnapshotExpiredRelease
 	for i := 0; i < len(guests); i += 1 {
 		// fake delete expired prepaid servers
 		if len(guests[i].ExternalId) > 0 {
@@ -4705,7 +4726,10 @@ func (manager *SGuestManager) DeleteExpiredPrepaidServers(ctx context.Context, u
 			}
 		}
 		guests[i].SetDisableDelete(userCred, false)
-		guests[i].StartDeleteGuestTask(ctx, userCred, "", false, false, deteleSnapshot)
+		opts := api.ServerDeleteInput{
+			DeleteSnapshots: deleteSnapshot,
+		}
+		guests[i].StartDeleteGuestTask(ctx, userCred, "", opts)
 	}
 }
 
@@ -4741,7 +4765,8 @@ func (manager *SGuestManager) DeleteExpiredPostpaidServers(ctx context.Context,
 			}
 		}
 		guests[i].SetDisableDelete(userCred, false)
-		guests[i].StartDeleteGuestTask(ctx, userCred, "", false, false, deleteSnapshot)
+		opts := api.ServerDeleteInput{DeleteSnapshots: deleteSnapshot}
+		guests[i].StartDeleteGuestTask(ctx, userCred, "", opts)
 	}
 }
 

pkg/compute/models/host_recycle.go

@@ -90,7 +90,10 @@ func (self *SGuest) DoPerformPrepaidRecycle(ctx context.Context, userCred mcclie
 	logclient.AddActionLogWithContext(ctx, self, logclient.ACT_RECYCLE_PREPAID, self.GetShortDesc(ctx), userCred, true)
 
 	if autoDelete {
-		self.StartDeleteGuestTask(ctx, userCred, "", false, true, false)
+		opts := api.ServerDeleteInput{
+			OverridePendingDelete: true,
+		}
+		self.StartDeleteGuestTask(ctx, userCred, "", opts)
 	}
 
 	return nil, nil

pkg/compute/models/hosts.go

@@ -2567,7 +2567,7 @@ func (manager *SHostManager) totalCountQ(
 		iq.Field("isolated_reserved_storage"),
 	)
 	q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
-	log.Debugf("hostCount: %s", q.String())
+	// log.Debugf("hostCount: %s", q.String())
 	return q
 }
 
@@ -3971,7 +3971,7 @@ func (self *SHost) AllowPerformIpmiProbe(ctx context.Context,
 }
 
 func (self *SHost) PerformIpmiProbe(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if utils.IsInStringArray(self.Status, []string{api.BAREMETAL_INIT, api.BAREMETAL_READY, api.BAREMETAL_RUNNING}) {
+	if utils.IsInStringArray(self.Status, []string{api.BAREMETAL_INIT, api.BAREMETAL_READY, api.BAREMETAL_RUNNING, api.BAREMETAL_PROBE_FAIL}) {
 		return nil, self.StartIpmiProbeTask(ctx, userCred, "")
 	}
 	return nil, httperrors.NewInvalidStatusError("Cannot do Ipmi-probe in status %s", self.Status)

pkg/compute/models/instance_snapshots.go

@@ -239,7 +239,7 @@ func (self *SInstanceSnapshot) getMoreDetails(userCred mcclient.TokenCredential,
 				out.StorageType = snapshots[i].GetStorageType()
 			}
 		}
-	} else {
+	} else if guest != nil {
 		out.Size = self.SizeMb
 		disk, err := guest.GetSystemDisk()
 		if err != nil {

pkg/compute/models/loadbalancerlisteners.go

@@ -816,13 +816,8 @@ func (lblis *SLoadbalancerListener) GetLoadbalancerCertificate() (*SCachedLoadba
 		return nil, nil
 	}
 
-	region := lblis.GetRegion()
-	if region == nil {
-		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "loadbalancer listener is not attached to any region")
-	}
-
 	ret := &SCachedLoadbalancerCertificate{}
-	err := CachedLoadbalancerCertificateManager.Query().Equals("id", lblis.CachedCertificateId).Equals("cloudregion_id", region.Id).IsFalse("pending_deleted").First(ret)
+	err := CachedLoadbalancerCertificateManager.Query().Equals("id", lblis.CachedCertificateId).IsFalse("pending_deleted").First(ret)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, nil

pkg/compute/models/networks.go

@@ -916,7 +916,7 @@ func (manager *SNetworkManager) TotalPortCount(
 	userCred mcclient.IIdentityProvider,
 	providers []string, brands []string, cloudEnv string,
 	rangeObjs []db.IStandaloneModel,
-) NetworkPortStat {
+) map[string]NetworkPortStat {
 	nets := make([]SNetwork, 0)
 	err := manager.totalPortCountQ(
 		scope,
@@ -927,17 +927,32 @@ func (manager *SNetworkManager) TotalPortCount(
 	if err != nil {
 		log.Errorf("TotalPortCount: %v", err)
 	}
-	ct := 0
-	ctExt := 0
+	ret := make(map[string]NetworkPortStat)
 	for _, net := range nets {
+		var stat NetworkPortStat
+		var allStat NetworkPortStat
+		if len(net.ServerType) > 0 {
+			stat, _ = ret[net.ServerType]
+		}
+		allStat, _ = ret[""]
 		count := net.getIPRange().AddressCount()
 		if net.IsExitNetwork() {
-			ctExt += count
+			if len(net.ServerType) > 0 {
+				stat.CountExt += count
+			}
+			allStat.CountExt += count
 		} else {
-			ct += count
+			if len(net.ServerType) > 0 {
+				stat.Count += count
+			}
+			allStat.Count += count
 		}
+		if len(net.ServerType) > 0 {
+			ret[net.ServerType] = stat
+		}
+		ret[""] = allStat
 	}
-	return NetworkPortStat{Count: ct, CountExt: ctExt}
+	return ret
 }
 
 type SNicConfig struct {
@@ -1042,9 +1057,9 @@ func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *api.Networ
 	return nil
 }
 
-func IsExitNetworkInfo(netConfig *api.NetworkConfig) bool {
+func IsExitNetworkInfo(userCred mcclient.TokenCredential, netConfig *api.NetworkConfig) bool {
 	if len(netConfig.Network) > 0 {
-		netObj, _ := NetworkManager.FetchById(netConfig.Network)
+		netObj, _ := NetworkManager.FetchByIdOrName(userCred, netConfig.Network)
 		net := netObj.(*SNetwork)
 		if net.IsExitNetwork() {
 			return true
@@ -1621,6 +1636,10 @@ func (self *SNetwork) validateUpdateData(ctx context.Context, userCred mcclient.
 		if endIp.NetAddr(masklen) != netAddr {
 			return input, httperrors.NewInputParameterError("start, end ip must be in the same subnet")
 		}
+	} else {
+		startIp, _ = netutils.NewIPV4Addr(self.GuestIpStart)
+		endIp, _ = netutils.NewIPV4Addr(self.GuestIpEnd)
+		netAddr = startIp.NetAddr(masklen)
 	}
 
 	for key, ipStr := range map[string]string{

pkg/compute/models/quotas.go

@@ -188,7 +188,7 @@ func (self *SQuota) FetchUsage(ctx context.Context) error {
 
 	diskSize := totalDiskSize(scope, ownerId, tristate.None, tristate.None, false, false, rangeObjs, providers, brands, keys.CloudEnv, hypervisors)
 
-	guest := usageTotalGuestResouceCount(scope, ownerId, rangeObjs, nil, hypervisors, false, false, nil, nil, providers, brands, keys.CloudEnv)
+	guest := usageTotalGuestResouceCount(scope, ownerId, rangeObjs, nil, hypervisors, false, false, nil, nil, providers, brands, keys.CloudEnv, nil)
 
 	self.Count = guest.TotalGuestCount
 	self.Cpu = guest.TotalCpuCount

pkg/compute/models/regiondrivers.go

@@ -159,14 +159,13 @@ type IDBInstanceDriver interface {
 	ValidateCreateDBInstanceAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, instance *SDBInstance, input api.DBInstanceAccountCreateInput) (api.DBInstanceAccountCreateInput, error)
 	ValidateCreateDBInstanceDatabaseData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, instance *SDBInstance, input api.DBInstanceDatabaseCreateInput) (api.DBInstanceDatabaseCreateInput, error)
 	ValidateCreateDBInstanceBackupData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, instance *SDBInstance, input api.DBInstanceBackupCreateInput) (api.DBInstanceBackupCreateInput, error)
-	ValidateChangeDBInstanceConfigData(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, input *api.SDBInstanceChangeConfigInput) error
 	ValidateDBInstanceAccountPrivilege(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, account string, privilege string) error
 	ValidateResetDBInstancePassword(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, account string) error
 
 	RequestCreateDBInstance(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *SDBInstance, task taskman.ITask) error
 	RequestCreateDBInstanceFromBackup(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *SDBInstance, task taskman.ITask) error
 	RequestCreateDBInstanceBackup(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, backup *SDBInstanceBackup, task taskman.ITask) error
-	RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, task taskman.ITask) error
+	RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, input *api.SDBInstanceChangeConfigInput, task taskman.ITask) error
 
 	IsSupportedDBInstance() bool
 	IsSupportedDBInstanceAutoRenew() bool
@@ -191,7 +190,7 @@ type IElasticcacheDriver interface {
 
 	AllowCreateElasticcacheBackup(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error
 	AllowUpdateElasticcacheAuthMode(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error
-	ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
+	ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error)
 	ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	ValidateCreateElasticcacheAclData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	ValidateCreateElasticcacheBackupData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)

pkg/compute/models/reservedips.go

@@ -167,8 +167,8 @@ func (manager *SReservedipManager) GetReservedIP(network *SNetwork, ip string) *
 
 func (manager *SReservedipManager) GetReservedIPs(network *SNetwork) []SReservedip {
 	rips := make([]SReservedip, 0)
-	now := time.Now().UTC()
-	q := manager.Query().Equals("network_id", network.Id).GT("expired_at", now)
+	q := manager.Query().Equals("network_id", network.Id)
+	q = filterExpiredReservedIps(q)
 	err := db.FetchModelObjects(manager, q, &rips)
 	if err != nil {
 		log.Errorf("GetReservedIPs fail: %s", err)

pkg/compute/models/schedtags.go

@@ -219,6 +219,13 @@ func (manager *SSchedtagManager) ListItemFilter(
 		q = q.In("default_strategy", query.DefaultStrategy)
 	}
 
+	if len(query.CloudproviderId) > 0 {
+		hostSubq := HostManager.Query("id").Equals("manager_id", query.CloudproviderId).SubQuery()
+		hostSchedtagQ := HostschedtagManager.Query("schedtag_id")
+		hostSchedtagSubq := hostSchedtagQ.Join(hostSubq, sqlchemy.Equals(hostSchedtagQ.Field("host_id"), hostSubq.Field("id"))).SubQuery()
+		q = q.Join(hostSchedtagSubq, sqlchemy.Equals(q.Field("id"), hostSchedtagSubq.Field("schedtag_id")))
+	}
+
 	return q, nil
 }
 

pkg/compute/models/storages.go

@@ -71,10 +71,10 @@ type SStorage struct {
 	SZoneResourceBase `update:""`
 
 	// 容量大小,单位Mb
-	Capacity int64 `nullable:"false" list:"domain" update:"domain" create:"domain_required"`
+	Capacity int64 `nullable:"false" list:"user" update:"domain" create:"domain_required"`
 	// 实际容量大小，单位Mb
 	// we always expect actual capacity great or equal than zero, otherwise something wrong
-	ActualCapacityUsed int64 `nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
+	ActualCapacityUsed int64 `nullable:"true" list:"user" update:"domain" create:"domain_optional"`
 	// 预留容量大小
 	Reserved int64 `nullable:"true" default:"0" list:"domain" update:"domain"`
 	// 存储类型

pkg/compute/models/vpc_peering_connections.go

@@ -143,6 +143,10 @@ func (manager *SVpcPeeringConnectionManager) ValidateCreateData(
 	}
 	peerVpc := _peerVpc.(*SVpc)
 
+	if len(vpc.ManagerId) == 0 || len(peerVpc.ManagerId) == 0 {
+		return input, httperrors.NewInputParameterError("Only public cloud support vpcpeering")
+	}
+
 	// get account,providerFactory
 	account := vpc.GetCloudaccount()
 	peerAccount := peerVpc.GetCloudaccount()

pkg/compute/models/wires.go

@@ -458,32 +458,57 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl
 	return &wire, nil
 }
 
-func filterByScopeOwnerId(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) *sqlchemy.SQuery {
+func filterByScopeOwnerId(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, domainResource bool) *sqlchemy.SQuery {
 	switch scope {
 	case rbacutils.ScopeSystem:
 	case rbacutils.ScopeDomain:
 		q = q.Equals("domain_id", ownerId.GetProjectDomainId())
 	case rbacutils.ScopeProject:
-		q = q.Equals("tenant_id", ownerId.GetProjectId())
+		if domainResource {
+			q = q.Equals("domain_id", ownerId.GetProjectId())
+		} else {
+			q = q.Equals("tenant_id", ownerId.GetProjectId())
+		}
 	}
 	return q
 }
 
+func fixVmwareProvider(providers []string) (bool, []string) {
+	findVmware := false
+	findOnecloud := false
+	newp := make([]string, 0)
+	for _, p := range providers {
+		if p == api.CLOUD_PROVIDER_VMWARE {
+			findVmware = true
+		} else {
+			if p == api.CLOUD_PROVIDER_ONECLOUD {
+				findOnecloud = true
+			}
+			newp = append(newp, p)
+		}
+	}
+	if findVmware && !findOnecloud {
+		newp = append(newp, api.CLOUD_PROVIDER_ONECLOUD)
+	}
+	return findVmware, newp
+}
+
 func (manager *SWireManager) totalCountQ(
 	rangeObjs []db.IStandaloneModel,
-	hostTypes []string,
+	hostTypes []string, hostProviders, hostBrands []string,
 	providers []string, brands []string, cloudEnv string,
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
 ) *sqlchemy.SQuery {
-	guestsQ := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId)
+	guestsQ := filterByScopeOwnerId(GuestManager.Query(), scope, ownerId, false)
 	guests := guestsQ.SubQuery()
 
+	// hosts no filter, for guest networks
 	hostsQ := HostManager.Query()
 	if len(hostTypes) > 0 {
 		hostsQ = hostsQ.In("host_type", hostTypes)
 	}
-	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+	if len(hostProviders) > 0 || len(hostBrands) > 0 || len(cloudEnv) > 0 {
 		hostsQ = CloudProviderFilter(hostsQ, hostsQ.Field("manager_id"), providers, brands, cloudEnv)
 	}
 	if len(rangeObjs) > 0 {
@@ -491,9 +516,23 @@ func (manager *SWireManager) totalCountQ(
 	}
 	hosts := hostsQ.SubQuery()
 
-	groups := filterByScopeOwnerId(GroupManager.Query(), scope, ownerId).SubQuery()
+	// hosts filter by owner, for host networks
+	hostsQ2 := HostManager.Query()
+	hostsQ2 = filterByScopeOwnerId(hostsQ2, scope, ownerId, true)
+	if len(hostTypes) > 0 {
+		hostsQ2 = hostsQ2.In("host_type", hostTypes)
+	}
+	if len(hostProviders) > 0 || len(hostBrands) > 0 || len(cloudEnv) > 0 {
+		hostsQ2 = CloudProviderFilter(hostsQ2, hostsQ2.Field("manager_id"), providers, brands, cloudEnv)
+	}
+	if len(rangeObjs) > 0 {
+		hostsQ2 = RangeObjectsFilter(hostsQ2, rangeObjs, nil, hostsQ.Field("zone_id"), hostsQ.Field("manager_id"), hostsQ.Field("id"), nil)
+	}
+	hosts2 := hostsQ2.SubQuery()
 
-	lbsQ := filterByScopeOwnerId(LoadbalancerManager.Query(), scope, ownerId)
+	groups := filterByScopeOwnerId(GroupManager.Query(), scope, ownerId, false).SubQuery()
+
+	lbsQ := filterByScopeOwnerId(LoadbalancerManager.Query(), scope, ownerId, false)
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		lbsQ = CloudProviderFilter(lbsQ, lbsQ.Field("manager_id"), providers, brands, cloudEnv)
 	}
@@ -502,7 +541,7 @@ func (manager *SWireManager) totalCountQ(
 	}
 	lbs := lbsQ.SubQuery()
 
-	dbsQ := filterByScopeOwnerId(DBInstanceManager.Query(), scope, ownerId)
+	dbsQ := filterByScopeOwnerId(DBInstanceManager.Query(), scope, ownerId, false)
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		dbsQ = CloudProviderFilter(dbsQ, dbsQ.Field("manager_id"), providers, brands, cloudEnv)
 	}
@@ -526,14 +565,8 @@ func (manager *SWireManager) totalCountQ(
 		hNics.Field("network_id"),
 		sqlchemy.COUNT("hnic_count"),
 	)
-	hNicQ = hNicQ.Join(hosts, sqlchemy.Equals(hNics.Field("baremetal_id"), hosts.Field("id")))
-	hNicQ = hNicQ.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-
-	revIps := ReservedipManager.Query().SubQuery()
-	revQ := revIps.Query(
-		revIps.Field("network_id"),
-		sqlchemy.COUNT("rnic_count"),
-	)
+	hNicQ = hNicQ.Join(hosts2, sqlchemy.Equals(hNics.Field("baremetal_id"), hosts2.Field("id")))
+	hNicQ = hNicQ.Filter(sqlchemy.IsTrue(hosts2.Field("enabled")))
 
 	groupNics := GroupnetworkManager.Query().SubQuery()
 	grpNicQ := groupNics.Query(
@@ -550,7 +583,8 @@ func (manager *SWireManager) totalCountQ(
 	lbNicQ = lbNicQ.Join(lbs, sqlchemy.Equals(lbs.Field("id"), lbNics.Field("loadbalancer_id")))
 	lbNicQ = lbNicQ.Filter(sqlchemy.IsFalse(lbs.Field("pending_deleted")))
 
-	eipNics := ElasticipManager.Query().IsNotEmpty("network_id").SubQuery()
+	eipNicsQ := ElasticipManager.Query().IsNotEmpty("network_id")
+	eipNics := filterByScopeOwnerId(eipNicsQ, scope, ownerId, false).SubQuery()
 	eipNicQ := eipNics.Query(
 		eipNics.Field("network_id"),
 		sqlchemy.COUNT("eipnic_count"),
@@ -563,6 +597,7 @@ func (manager *SWireManager) totalCountQ(
 	}
 
 	netifsQ := NetworkInterfaceManager.Query()
+	netifsQ = filterByScopeOwnerId(netifsQ, scope, ownerId, true)
 	if len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		netifsQ = CloudProviderFilter(netifsQ, netifsQ.Field("manager_id"), providers, brands, cloudEnv)
 	}
@@ -587,7 +622,6 @@ func (manager *SWireManager) totalCountQ(
 
 	gNicSQ := gNicQ.GroupBy(gNics.Field("network_id")).SubQuery()
 	hNicSQ := hNicQ.GroupBy(hNics.Field("network_id")).SubQuery()
-	revSQ := revQ.GroupBy(revIps.Field("network_id")).SubQuery()
 	grpNicSQ := grpNicQ.GroupBy(groupNics.Field("network_id")).SubQuery()
 	lbNicSQ := lbNicQ.GroupBy(lbNics.Field("network_id")).SubQuery()
 	eipNicSQ := eipNicQ.GroupBy(eipNics.Field("network_id")).SubQuery()
@@ -596,58 +630,90 @@ func (manager *SWireManager) totalCountQ(
 
 	networks := NetworkManager.Query().SubQuery()
 	netQ := networks.Query(
-		networks.Field("wire_id"),
-		sqlchemy.COUNT("id").Label("net_count"),
-		sqlchemy.SUM("gnic_count", gNicSQ.Field("gnic_count")),
-		sqlchemy.SUM("pending_deleted_gnic_count", gNicSQ.Field("pending_deleted_gnic_count")),
-		sqlchemy.SUM("hnic_count", hNicSQ.Field("hnic_count")),
-		sqlchemy.SUM("rev_count", revSQ.Field("rnic_count")),
-		sqlchemy.SUM("grpnic_count", grpNicSQ.Field("grpnic_count")),
-		sqlchemy.SUM("lbnic_count", lbNicSQ.Field("lbnic_count")),
-		sqlchemy.SUM("eipnic_count", eipNicSQ.Field("eipnic_count")),
-		sqlchemy.SUM("netifnic_count", netifNicSQ.Field("netifnic_count")),
-		sqlchemy.SUM("dbnic_count", dbNicSQ.Field("dbnic_count")),
+		sqlchemy.SUM("guest_nic_count", gNicSQ.Field("gnic_count")),
+		sqlchemy.SUM("pending_deleted_guest_nic_count", gNicSQ.Field("pending_deleted_gnic_count")),
+		sqlchemy.SUM("host_nic_count", hNicSQ.Field("hnic_count")),
+		sqlchemy.SUM("group_nic_count", grpNicSQ.Field("grpnic_count")),
+		sqlchemy.SUM("lb_nic_count", lbNicSQ.Field("lbnic_count")),
+		sqlchemy.SUM("eip_nic_count", eipNicSQ.Field("eipnic_count")),
+		sqlchemy.SUM("netif_nic_count", netifNicSQ.Field("netifnic_count")),
+		sqlchemy.SUM("db_nic_count", dbNicSQ.Field("dbnic_count")),
 	)
 	netQ = netQ.LeftJoin(gNicSQ, sqlchemy.Equals(gNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(hNicSQ, sqlchemy.Equals(hNicSQ.Field("network_id"), networks.Field("id")))
-	netQ = netQ.LeftJoin(revSQ, sqlchemy.Equals(revSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(grpNicSQ, sqlchemy.Equals(grpNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(lbNicSQ, sqlchemy.Equals(lbNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(eipNicSQ, sqlchemy.Equals(eipNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(netifNicSQ, sqlchemy.Equals(netifNicSQ.Field("network_id"), networks.Field("id")))
 	netQ = netQ.LeftJoin(dbNicSQ, sqlchemy.Equals(dbNicSQ.Field("network_id"), networks.Field("id")))
 
-	netQ = netQ.GroupBy(networks.Field("wire_id"))
-	netSQ := netQ.SubQuery()
+	return netQ
+}
+
+func (manager *SWireManager) totalCountQ2(
+	rangeObjs []db.IStandaloneModel,
+	hostTypes []string,
+	providers []string, brands []string, cloudEnv string,
+	scope rbacutils.TRbacScope,
+	ownerId mcclient.IIdentityProvider,
+) *sqlchemy.SQuery {
+	revIps := filterExpiredReservedIps(ReservedipManager.Query()).SubQuery()
+	revQ := revIps.Query(
+		revIps.Field("network_id"),
+		sqlchemy.COUNT("rnic_count"),
+	)
+
+	revSQ := revQ.GroupBy(revIps.Field("network_id")).SubQuery()
+
+	ownerNetworks := filterByScopeOwnerId(NetworkManager.Query(), scope, ownerId, false).SubQuery()
+	ownerNetQ := ownerNetworks.Query(
+		ownerNetworks.Field("wire_id"),
+		sqlchemy.COUNT("id").Label("net_count"),
+		sqlchemy.SUM("rev_count", revSQ.Field("rnic_count")),
+	)
+	ownerNetQ = ownerNetQ.LeftJoin(revSQ, sqlchemy.Equals(revSQ.Field("network_id"), ownerNetworks.Field("id")))
+	ownerNetQ = ownerNetQ.GroupBy(ownerNetworks.Field("wire_id"))
+	ownerNetSQ := ownerNetQ.SubQuery()
 
 	wires := WireManager.Query().SubQuery()
+	q := wires.Query(
+		sqlchemy.SUM("net_count", ownerNetSQ.Field("net_count")),
+		sqlchemy.SUM("reserved_count", ownerNetSQ.Field("rev_count")),
+	)
+	q = q.LeftJoin(ownerNetSQ, sqlchemy.Equals(wires.Field("id"), ownerNetSQ.Field("wire_id")))
+	return filterWiresCountQuery(q, hostTypes, providers, brands, cloudEnv, rangeObjs)
+}
+
+func (manager *SWireManager) totalCountQ3(
+	rangeObjs []db.IStandaloneModel,
+	hostTypes []string,
+	providers []string, brands []string, cloudEnv string,
+	scope rbacutils.TRbacScope,
+	ownerId mcclient.IIdentityProvider,
+) *sqlchemy.SQuery {
+	wires := filterByScopeOwnerId(WireManager.Query(), scope, ownerId, true).SubQuery()
 	q := wires.Query(
 		sqlchemy.COUNT("id").Label("wires_count"),
 		sqlchemy.SUM("emulated_wires_count", wires.Field("is_emulated")),
-		sqlchemy.SUM("net_count", netSQ.Field("net_count")),
-		sqlchemy.SUM("guest_nic_count", netSQ.Field("gnic_count")),
-		sqlchemy.SUM("pending_deleted_guest_nic_count", netSQ.Field("pending_deleted_gnic_count")),
-		sqlchemy.SUM("host_nic_count", netSQ.Field("hnic_count")),
-		sqlchemy.SUM("reserved_count", netSQ.Field("rev_count")),
-		sqlchemy.SUM("group_nic_count", netSQ.Field("grpnic_count")),
-		sqlchemy.SUM("lb_nic_count", netSQ.Field("lbnic_count")),
-		sqlchemy.SUM("eip_nic_count", netSQ.Field("eipnic_count")),
-		sqlchemy.SUM("netif_nic_count", netSQ.Field("netifnic_count")),
-		sqlchemy.SUM("db_nic_count", netSQ.Field("dbnic_count")),
 	)
-	q = q.LeftJoin(netSQ, sqlchemy.Equals(wires.Field("id"), netSQ.Field("wire_id")))
+	return filterWiresCountQuery(q, hostTypes, providers, brands, cloudEnv, rangeObjs)
+}
 
+func filterWiresCountQuery(q *sqlchemy.SQuery, hostTypes, providers, brands []string, cloudEnv string, rangeObjs []db.IStandaloneModel) *sqlchemy.SQuery {
 	if len(hostTypes) > 0 {
 		hostwires := HostwireManager.Query().SubQuery()
 		hosts := HostManager.Query().SubQuery()
-		q = q.Join(hostwires, sqlchemy.Equals(q.Field("id"), hostwires.Field("wire_id")))
-		q = q.Join(hosts, sqlchemy.Equals(hostwires.Field("host_id"), hosts.Field("id")))
-		q = q.Filter(sqlchemy.In(hosts.Field("host_type"), hostTypes))
+		hostWireQ := hostwires.Query(hostwires.Field("wire_id"))
+		hostWireQ = hostWireQ.Join(hosts, sqlchemy.Equals(hostWireQ.Field("host_id"), hosts.Field("id")))
+		hostWireQ = hostWireQ.Filter(sqlchemy.In(hosts.Field("host_type"), hostTypes))
+		hostWireQ = hostWireQ.GroupBy(hostwires.Field("wire_id"))
+		hostWireSQ := hostWireQ.SubQuery()
+
+		q = q.Join(hostWireSQ, sqlchemy.Equals(hostWireSQ.Field("wire_id"), q.Field("id")))
 	}
 
 	if len(rangeObjs) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
 		vpcs := VpcManager.Query().SubQuery()
-
 		q = q.Join(vpcs, sqlchemy.Equals(q.Field("vpc_id"), vpcs.Field("id")))
 		q = CloudProviderFilter(q, vpcs.Field("manager_id"), providers, brands, cloudEnv)
 		q = RangeObjectsFilter(q, rangeObjs, vpcs.Field("cloudregion_id"), q.Field("zone_id"), vpcs.Field("manager_id"), nil, nil)
@@ -683,15 +749,60 @@ func (manager *SWireManager) TotalCount(
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
 ) WiresCountStat {
+	vmwareP, hostProviders := fixVmwareProvider(providers)
+	vmwareB, hostBrands := fixVmwareProvider(brands)
+
+	if vmwareP || vmwareB {
+		if !utils.IsInStringArray(api.HOST_TYPE_ESXI, hostTypes) {
+			hostTypes = append(hostTypes, api.HOST_TYPE_ESXI)
+		}
+	} else {
+		if utils.IsInStringArray(api.HOST_TYPE_ESXI, hostTypes) {
+			providers = append(providers, api.CLOUD_PROVIDER_VMWARE)
+			brands = append(brands, api.CLOUD_PROVIDER_VMWARE)
+		}
+	}
+	if len(hostTypes) > 0 {
+		for _, p := range providers {
+			if hs, ok := api.CLOUD_PROVIDER_HOST_TYPE_MAP[p]; ok {
+				hostTypes = append(hostTypes, hs...)
+			}
+		}
+		for _, p := range brands {
+			if hs, ok := api.CLOUD_PROVIDER_HOST_TYPE_MAP[p]; ok {
+				hostTypes = append(hostTypes, hs...)
+			}
+		}
+	}
+	log.Debugf("providers: %#v hostProviders: %#v brands: %#v hostBrands: %#v hostTypes: %#v", providers, hostProviders, brands, hostBrands, hostTypes)
+
 	stat := WiresCountStat{}
 	err := manager.totalCountQ(
+		rangeObjs,
+		hostTypes, hostProviders, hostBrands,
+		providers, brands, cloudEnv,
+		scope, ownerId,
+	).First(&stat)
+	if err != nil {
+		log.Errorf("Wire total count: %v", err)
+	}
+	err = manager.totalCountQ2(
 		rangeObjs,
 		hostTypes,
 		providers, brands, cloudEnv,
 		scope, ownerId,
 	).First(&stat)
 	if err != nil {
-		log.Errorf("Wire total count: %v", err)
+		log.Errorf("Wire total count 2: %v", err)
+	}
+	err = manager.totalCountQ3(
+		rangeObjs,
+		hostTypes,
+		providers, brands, cloudEnv,
+		scope, ownerId,
+	).First(&stat)
+	if err != nil {
+		log.Errorf("Wire total count 2: %v", err)
 	}
 	return stat
 }

pkg/compute/options/options.go

@@ -121,8 +121,8 @@ type ComputeOptions struct {
 
 	CloudSyncWorkerCount         int `help:"how many current synchronization threads" default:"5"`
 	CloudAutoSyncIntervalSeconds int `help:"frequency to check auto sync tasks" default:"30"`
-	DefaultSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"900"`
-	MinimalSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"300"`
+	DefaultSyncIntervalSeconds   int `help:"minimal synchronization interval, default 15 minutes" default:"900"`
+	MinimalSyncIntervalSeconds   int `help:"minimal synchronization interval, default 30 minutes" default:"1800"`
 	MaxCloudAccountErrorCount    int `help:"maximal consecutive error count allow for a cloud account" default:"5"`
 
 	NameSyncResources []string `help:"resources that need synchronization of name"`

pkg/compute/regiondrivers/aliyun.go

@@ -1171,7 +1171,8 @@ func (self *SAliyunRegionDriver) ValidateDBInstanceAccountPrivilege(ctx context.
 	return nil
 }
 
-func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
+	data := input.JSON(input)
 	zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
 	networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
 

pkg/compute/regiondrivers/base.go

@@ -319,7 +319,7 @@ func (self *SBaseRegionDriver) InitDBInstanceUser(ctx context.Context, dbinstanc
 	return nil
 }
 
-func (self *SBaseRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, task taskman.ITask) error {
+func (self *SBaseRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, input *api.SDBInstanceChangeConfigInput, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement RequestChangeDBInstanceConfig")
 }
 
@@ -335,10 +335,6 @@ func (self *SBaseRegionDriver) ValidateCreateDBInstanceBackupData(ctx context.Co
 	return input, fmt.Errorf("Not Implement ValidateCreateDBInstanceBackupData")
 }
 
-func (self *SBaseRegionDriver) ValidateChangeDBInstanceConfigData(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, input *api.SDBInstanceChangeConfigInput) error {
-	return fmt.Errorf("Not Implement ValidateChangeDBInstanceConfigData")
-}
-
 func (self *SBaseRegionDriver) ValidateDBInstanceAccountPrivilege(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, account string, privilege string) error {
 	return fmt.Errorf("Not Implement ValidateDBInstanceAccountPrivilege")
 }

pkg/compute/regiondrivers/huawei.go

@@ -2247,12 +2247,6 @@ func (self *SHuaWeiRegionDriver) ValidateChangeDBInstanceConfigData(ctx context.
 	if input.DiskSizeGB != 0 && input.DiskSizeGB < instance.DiskSizeGB {
 		return httperrors.NewUnsupportOperationError("Huawei DBInstance Disk cannot be thrink")
 	}
-	if len(input.Category) > 0 && input.Category != instance.Category {
-		return httperrors.NewUnsupportOperationError("Huawei DBInstance category cannot change")
-	}
-	if len(input.StorageType) > 0 && input.StorageType != instance.StorageType {
-		return httperrors.NewUnsupportOperationError("Huawei DBInstance storage type cannot change")
-	}
 	return nil
 }
 
@@ -2301,7 +2295,7 @@ func (self *SHuaWeiRegionDriver) ValidateDBInstanceRecovery(ctx context.Context,
 	return nil
 }
 
-func validatorSlaveZones(ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict, optional bool) error {
+func validatorSlaveZones(ownerId mcclient.IIdentityProvider, regionId string, data *jsonutils.JSONDict, optional bool) error {
 	s, err := data.GetString("slave_zones")
 	if err != nil {
 		if optional {
@@ -2314,33 +2308,36 @@ func validatorSlaveZones(ownerId mcclient.IIdentityProvider, data *jsonutils.JSO
 	zones := strings.Split(s, ",")
 	ret := []string{}
 	zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
-	for _, zone := range zones {
+	for i := range zones {
 		_data := jsonutils.NewDict()
-		_data.Add(jsonutils.NewString(zone), "zone")
+		_data.Set("zone", jsonutils.NewString(zones[i]))
 		if err := zoneV.Validate(_data); err != nil {
 			return errors.Wrap(err, "validatorSlaveZones")
 		} else {
+			if zoneV.Model.(*models.SZone).GetCloudRegionId() != regionId {
+				return errors.Wrap(fmt.Errorf("zone %s is not in region %s", zoneV.Model.GetName(), regionId), "GetCloudRegionId")
+			}
 			ret = append(ret, zoneV.Model.GetId())
 		}
 	}
 
-	if sku, err := data.GetString("sku"); err != nil || len(sku) == 0 {
-		return httperrors.NewMissingParameterError("sku")
-	} else {
-		chargeType, _ := data.GetString("charge_type")
-
-		_skuModel, err := db.FetchByIdOrName(models.ElasticcacheSkuManager, ownerId, sku)
-		if err != nil {
-			return err
-		}
-
-		skuModel := _skuModel.(*models.SElasticcacheSku)
-		for _, zoneId := range zones {
-			if err := ValidateElasticcacheSku(zoneId, chargeType, skuModel, nil); err != nil {
-				return err
-			}
-		}
-	}
+	//if sku, err := data.GetString("sku"); err != nil || len(sku) == 0 {
+	//	return httperrors.NewMissingParameterError("sku")
+	//} else {
+	//	chargeType, _ := data.GetString("charge_type")
+	//
+	//	_skuModel, err := db.FetchByIdOrName(models.ElasticcacheSkuManager, ownerId, sku)
+	//	if err != nil {
+	//		return err
+	//	}
+	//
+	//	skuModel := _skuModel.(*models.SElasticcacheSku)
+	//	for _, zoneId := range zones {
+	//		if err := ValidateElasticcacheSku(zoneId, chargeType, skuModel, nil); err != nil {
+	//			return err
+	//		}
+	//	}
+	//}
 
 	data.Set("slave_zones", jsonutils.NewString(strings.Join(ret, ",")))
 	return nil
@@ -2370,7 +2367,8 @@ func ValidateElasticcacheSku(zoneId string, chargeType string, sku *models.SElas
 	return nil
 }
 
-func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
+	data := input.JSON(input)
 	zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
 	networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
 	// secgroupV := validators.NewModelIdOrNameValidator("security_group", "secgroup", ownerId)
@@ -2423,7 +2421,10 @@ func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Cont
 	}
 
 	// validate slave zones
-	if err := validatorSlaveZones(ownerId, data, true); err != nil {
+	if len(input.SlaveZones) > 0 {
+		data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ",")))
+	}
+	if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil {
 		return nil, err
 	}
 

pkg/compute/regiondrivers/kvm.go

@@ -1270,8 +1270,8 @@ func (self *SKVMRegionDriver) RequestCreateElasticcache(ctx context.Context, use
 	return nil
 }
 
-func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	return data, nil
+func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
+	return input.JSON(input), nil
 }
 
 func (self *SKVMRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask) error {

pkg/compute/regiondrivers/managedvirtual.go

@@ -1914,8 +1914,15 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateElasticcache(ctx co
 	return nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	return self.ValidateManagerId(ctx, userCred, data)
+func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
+	m := jsonutils.NewDict()
+	m.Set("manager_id", jsonutils.NewString(input.ManagerId))
+	_, err := self.ValidateManagerId(ctx, userCred, m)
+	if err != nil {
+		return nil, errors.Wrap(err, "ValidateManagerId")
+	}
+
+	return input.JSON(input), nil
 }
 
 func (self *SManagedVirtualizationRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask) error {
@@ -2461,74 +2468,27 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateElasticcacheAcl(ctx
 	return nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, task taskman.ITask) error {
+func (self *SManagedVirtualizationRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, rds *models.SDBInstance, input *api.SDBInstanceChangeConfigInput, task taskman.ITask) error {
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
-		input := &api.SDBInstanceChangeConfigInput{}
-		err := task.GetParams().Unmarshal(input)
+		conf := cloudprovider.SManagedDBInstanceChangeConfig{}
+
+		if input.DiskSizeGB > 0 && input.DiskSizeGB != rds.DiskSizeGB {
+			conf.DiskSizeGB = input.DiskSizeGB
+		}
+
+		if len(input.InstanceType) > 0 && input.InstanceType != rds.InstanceType {
+			conf.InstanceType = input.InstanceType
+		}
+
+		iRds, err := rds.GetIDBInstance()
 		if err != nil {
-			return nil, errors.Wrap(err, "task.GetParams().Unmarshal")
-		}
-		if len(input.StorageType) > 0 {
-			instance.StorageType = input.StorageType
+			return nil, errors.Wrap(err, "rds.GetIDBInstance")
 		}
 
-		conf := cloudprovider.SManagedDBInstanceChangeConfig{
-			DiskSizeGB:  input.DiskSizeGB,
-			StorageType: instance.StorageType,
-		}
-
-		opts := []cloudprovider.SManagedDBInstanceChangeConfig{}
-
-		if len(input.InstanceType) > 0 || input.VCpuCount > 0 || input.VmemSizeMb > 0 {
-			instance.InstanceType = input.InstanceType
-			if input.VCpuCount > 0 {
-				instance.VcpuCount = input.VCpuCount
-			}
-			if input.VmemSizeMb > 0 {
-				instance.VmemSizeMb = input.VmemSizeMb
-			}
-			skus, err := instance.GetDBInstanceSkus()
-			if err != nil {
-				return nil, errors.Wrap(err, "instance.GetDBInstanceSkus")
-			}
-			for i := range skus {
-				conf.InstanceType = skus[i].Name
-				conf.VcpuCount = skus[i].VcpuCount
-				conf.VmemSizeMb = skus[i].VmemSizeMb
-				opts = append(opts, conf)
-			}
-		} else {
-			conf.InstanceType = instance.InstanceType
-			conf.VcpuCount = instance.VcpuCount
-			conf.VmemSizeMb = instance.VmemSizeMb
-			opts = append(opts, conf)
-		}
-
-		iRds, err := instance.GetIDBInstance()
+		log.Infof("change config: %s", jsonutils.Marshal(conf).String())
+		err = iRds.ChangeConfig(ctx, &conf)
 		if err != nil {
-			return nil, errors.Wrap(err, "instance.GetIDBInstance")
-		}
-
-		var changeConfig = func() error {
-			errMsgs := []string{}
-			for i := range opts {
-				log.Infof("change config: %s", jsonutils.Marshal(opts[i]).String())
-				err = iRds.ChangeConfig(ctx, &opts[i])
-				if err != nil {
-					errMsgs = append(errMsgs, err.Error())
-					continue
-				}
-				return nil
-			}
-			if len(errMsgs) > 0 {
-				return fmt.Errorf(strings.Join(errMsgs, "\n"))
-			}
-			return fmt.Errorf("no available dbinstance sku to change")
-		}
-
-		err = changeConfig()
-		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "iRds.ChangeConfig")
 		}
 
 		err = cloudprovider.WaitStatus(iRds, api.DBINSTANCE_RUNNING, time.Second*10, time.Minute*40)
@@ -2541,13 +2501,13 @@ func (self *SManagedVirtualizationRegionDriver) RequestChangeDBInstanceConfig(ct
 			return nil, errors.Wrapf(err, "iRds.Refresh")
 		}
 
-		_, err = db.Update(instance, func() error {
-			instance.InstanceType = iRds.GetInstanceType()
-			instance.Category = iRds.GetCategory()
-			instance.VcpuCount = iRds.GetVcpuCount()
-			instance.VmemSizeMb = iRds.GetVmemSizeMB()
-			instance.StorageType = iRds.GetStorageType()
-			instance.DiskSizeGB = iRds.GetDiskSizeGB()
+		_, err = db.Update(rds, func() error {
+			rds.InstanceType = iRds.GetInstanceType()
+			rds.Category = iRds.GetCategory()
+			rds.VcpuCount = iRds.GetVcpuCount()
+			rds.VmemSizeMb = iRds.GetVmemSizeMB()
+			rds.StorageType = iRds.GetStorageType()
+			rds.DiskSizeGB = iRds.GetDiskSizeGB()
 			return nil
 		})
 		if err != nil {
@@ -2610,10 +2570,6 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstanceBackup(ct
 	return nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) ValidateChangeDBInstanceConfigData(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, input *api.SDBInstanceChangeConfigInput) error {
-	return nil
-}
-
 func (self *SManagedVirtualizationRegionDriver) ValidateResetDBInstancePassword(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, account string) error {
 	return nil
 }

pkg/compute/regiondrivers/qcloud.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"regexp"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -1465,7 +1466,8 @@ func (self *SQcloudRegionDriver) GetMaxElasticcacheSecurityGroupCount() int {
 	return 10
 }
 
-func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) {
+	data := input.JSON(input)
 	zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId)
 	networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId)
 	instanceTypeV := validators.NewModelIdOrNameValidator("instance_type", "elasticcachesku", ownerId)
@@ -1507,6 +1509,28 @@ func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Cont
 		data.Set("capacity_mb", jsonutils.NewInt(int64(sku.MemorySizeMB)))
 	}
 
+	// validate slave zones
+	if len(input.SlaveZones) > 0 {
+		if len(input.SlaveZones) < sku.ReplicasNum {
+			padding := make([]string, sku.ReplicasNum-len(input.SlaveZones))
+			for i := range padding {
+				padding[i] = zoneId
+			}
+
+			input.SlaveZones = append(input.SlaveZones, padding...)
+		}
+
+		data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ",")))
+	}
+	if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil {
+		return nil, err
+	}
+
+	sz, _ := data.GetString("slave_zones")
+	if len(strings.Split(sz, ",")) > sku.ReplicasNum {
+		return nil, fmt.Errorf("the number of slave zones can not beyond redis replicas number")
+	}
+
 	// validate secgroups
 	secgroups := []string{}
 	err := data.Unmarshal(&secgroups, "secgroup_ids")

pkg/compute/tasks/baremetal_convert_hypervisor_task.go

@@ -90,7 +90,8 @@ func (self *BaremetalConvertHypervisorTask) OnGuestDeployCompleteFailed(ctx cont
 	guest := self.getGuest()
 	guest.SetDisableDelete(self.UserCred, false)
 	self.SetStage("OnGuestDeleteComplete", nil)
-	guest.StartDeleteGuestTask(ctx, self.UserCred, self.GetTaskId(), false, true, false)
+	opts := api.ServerDeleteInput{OverridePendingDelete: true}
+	guest.StartDeleteGuestTask(ctx, self.UserCred, self.GetTaskId(), opts)
 	logclient.AddActionLogWithStartable(self, baremetal, logclient.ACT_BM_CONVERT_HYPER, fmt.Sprintf("convert deploy failed: %s", body.String()), self.UserCred, false)
 }
 

pkg/compute/tasks/baremetal_unconvert_hypervisor_task.go

@@ -51,7 +51,8 @@ func (self *BaremetalUnconvertHypervisorTask) OnInit(ctx context.Context, obj db
 	if len(guests) == 1 {
 		guest := guests[0]
 		self.SetStage("OnGuestDeleteComplete", nil)
-		guest.StartDeleteGuestTask(ctx, self.UserCred, self.GetTaskId(), false, true, false)
+		opts := api.ServerDeleteInput{OverridePendingDelete: true}
+		guest.StartDeleteGuestTask(ctx, self.UserCred, self.GetTaskId(), opts)
 	} else {
 		self.OnGuestDeleteComplete(ctx, baremetal, nil)
 	}

pkg/compute/tasks/dbinstance_change_config.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -44,11 +45,18 @@ func (self *DBInstanceChangeConfigTask) taskFailed(ctx context.Context, dbinstan
 }
 
 func (self *DBInstanceChangeConfigTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
-	instance := obj.(*models.SDBInstance)
+	rds := obj.(*models.SDBInstance)
 	self.SetStage("OnDBInstanceChangeConfigComplete", nil)
-	err := instance.GetRegion().GetDriver().RequestChangeDBInstanceConfig(ctx, self.UserCred, instance, self)
+	input := &api.SDBInstanceChangeConfigInput{}
+	err := self.GetParams().Unmarshal(input)
 	if err != nil {
-		self.taskFailed(ctx, instance, err)
+		self.taskFailed(ctx, rds, errors.Wrapf(err, "GetParams().Unmarshal"))
+		return
+	}
+
+	err = rds.GetRegion().GetDriver().RequestChangeDBInstanceConfig(ctx, self.UserCred, rds, input, self)
+	if err != nil {
+		self.taskFailed(ctx, rds, err)
 		return
 	}
 }

pkg/compute/tasks/guest_delete_task.go

@@ -127,7 +127,8 @@ func (self *GuestDeleteTask) OnStartEipDissociate(ctx context.Context, guest *mo
 			self.OnEipDissociateComplete(ctx, guest, nil)
 		} else {
 			self.SetStage("OnEipDissociateComplete", nil)
-			eip.StartEipDissociateTask(ctx, self.UserCred, false, self.GetTaskId())
+			autoDelete := jsonutils.QueryBoolean(self.GetParams(), "delete_eip", false)
+			eip.StartEipDissociateTask(ctx, self.UserCred, autoDelete, self.GetTaskId())
 		}
 	} else {
 		self.OnEipDissociateComplete(ctx, guest, nil)
@@ -157,6 +158,10 @@ func (self *GuestDeleteTask) OnDiskDetachComplete(ctx context.Context, obj db.IS
 	}
 	// detach last detachable disk
 	lastDisk := guestdisks[len(guestdisks)-1].GetDisk()
+	deleteDisks := jsonutils.QueryBoolean(self.Params, "delete_disks", false)
+	if deleteDisks {
+		lastDisk.SetAutoDelete(lastDisk, self.GetUserCred(), true)
+	}
 	log.Debugf("lastDisk IsDetachable?? %v", lastDisk.IsDetachable())
 	if !lastDisk.IsDetachable() {
 		// no more disk need detach

pkg/compute/tasks/guest_detach_scalinggroup.go

@@ -98,7 +98,9 @@ func (self *GuestDetachScalingGroupTask) OnDetachLoadbalancerComplete(ctx contex
 	}
 	self.Params.Set("guest_name", jsonutils.NewString(guest.GetName()))
 	self.SetStage("OnDeleteGuestComplete", nil)
-	if err := guest.StartDeleteGuestTask(ctx, self.UserCred, self.Id, false, true, true); err != nil {
+	opts := api.ServerDeleteInput{Purge: false, OverridePendingDelete: true, DeleteSnapshots: true}
+	err := guest.StartDeleteGuestTask(ctx, self.UserCred, self.Id, opts)
+	if err != nil {
 		self.taskFailed(ctx, sg, nil, jsonutils.NewString(err.Error()))
 	}
 }

pkg/compute/usages/cache.go

@@ -0,0 +1,85 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package usages
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/hashcache"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
+
+var (
+	usageCache = hashcache.NewCache(1024, time.Second*300) // 5 minutes, 1024 buckets cache
+)
+
+func getCacheKey(
+	scope rbacutils.TRbacScope,
+	userCred mcclient.IIdentityProvider,
+	isOwner bool,
+	rangeObjs []db.IStandaloneModel,
+	hostTypes []string,
+	providers []string,
+	brands []string,
+	cloudEnv string,
+	includeSystem bool,
+) string {
+	type RangeObject struct {
+		Resource string `json:"resource"`
+		Id       string `json:"id"`
+	}
+	type KeyStruct struct {
+		Scope     rbacutils.TRbacScope `json:"scope"`
+		Domain    string               `json:"domain"`
+		Project   string               `json:"project"`
+		IsOwner   bool                 `json:"is_owner"`
+		Ranges    []RangeObject        `json:"ranges"`
+		HostTypes []string             `json:"host_types"`
+		Providers []string             `json:"providers"`
+		Brands    []string             `json:"brands"`
+		CloudEnv  string               `json:"cloud_env"`
+		System    bool                 `json:"system"`
+	}
+	key := KeyStruct{}
+	key.Scope = scope
+	switch scope {
+	case rbacutils.ScopeSystem:
+	case rbacutils.ScopeDomain:
+		key.Domain = userCred.GetProjectDomainId()
+	case rbacutils.ScopeProject:
+		key.Project = userCred.GetProjectId()
+	}
+	if isOwner {
+		key.IsOwner = true
+	}
+	for _, obj := range rangeObjs {
+		robj := RangeObject{
+			Resource: obj.Keyword(),
+			Id:       obj.GetId(),
+		}
+		key.Ranges = append(key.Ranges, robj)
+	}
+	key.HostTypes = hostTypes
+	key.Providers = providers
+	key.Brands = brands
+	key.CloudEnv = cloudEnv
+	key.System = includeSystem
+	jsonObj := jsonutils.Marshal(key)
+	return jsonObj.QueryString()
+}

pkg/compute/usages/handler.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	json "yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -121,11 +122,21 @@ func rangeObjHandler(
 		if obj != nil {
 			rangeObjs = []db.IStandaloneModel{obj}
 		}
+		refresh := json.QueryBoolean(query, "refresh", false)
+		key := getCacheKey(scope, ownerId, isOwner, rangeObjs, hostTypes, providers, brands, cloudEnv, includeSystem)
+		if !refresh {
+			cached := usageCache.Get(key)
+			if cached != nil {
+				response(w, cached)
+				return
+			}
+		}
 		usage, err := reporter(scope, ownerId, isOwner, rangeObjs, hostTypes, providers, brands, cloudEnv, includeSystem)
 		if err != nil {
 			httperrors.GeneralServerError(ctx, w, err)
 			return
 		}
+		usageCache.AtomicSet(key, usage)
 		response(w, usage)
 	}
 }
@@ -241,49 +252,57 @@ func getSystemGeneralUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.I
 	count.Add("all.memory_commit_rate.running", runningMemCmtRate)
 	count.Add("all.cpu_commit_rate.running", runningCpuCmtRate)
 
+	lastWeek := time.Now().Add(-7 * 24 * time.Hour)
 	count.Include(
 		VpcUsage("", providers, brands, cloudEnv, nil, rbacutils.ScopeSystem, rangeObjs),
 
 		DnsZoneUsage("", nil, rbacutils.ScopeSystem),
 
 		HostAllUsage("", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv),
-		HostAllUsage("prepaid_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
-		HostAllUsage("any_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
+		// HostAllUsage("prepaid_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
+		// HostAllUsage("any_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
 
 		hostEnabledUsage,
-		HostEnabledUsage("prepaid_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
-		HostEnabledUsage("any_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
+		// HostEnabledUsage("prepaid_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
+		// HostEnabledUsage("any_pool", userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
 
 		BaremetalUsage(userCred, rbacutils.ScopeSystem, rangeObjs, hostTypes, providers, brands, cloudEnv),
 
 		StorageUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, includeSystem, rbacutils.ScopeSystem, nil),
 		StorageUsage("system", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, true, rbacutils.ScopeSystem, nil),
-		StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false, includeSystem, rbacutils.ScopeSystem, nil),
-		StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, includeSystem, rbacutils.ScopeSystem, nil),
-		StorageUsage("any_pool.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, true, rbacutils.ScopeSystem, nil),
-		StorageUsage("any_pool.pending_delete", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, includeSystem, rbacutils.ScopeSystem, nil),
-		StorageUsage("any_pool.pending_delete.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, true, rbacutils.ScopeSystem, nil),
+		// StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false, includeSystem, rbacutils.ScopeSystem, nil),
+		// StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, includeSystem, rbacutils.ScopeSystem, nil),
+		// StorageUsage("any_pool.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, true, rbacutils.ScopeSystem, nil),
+		// StorageUsage("any_pool.pending_delete", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, includeSystem, rbacutils.ScopeSystem, nil),
+		// StorageUsage("any_pool.pending_delete.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, true, rbacutils.ScopeSystem, nil),
 
-		GuestNormalUsage("all.servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem),
-		GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
-		GuestNormalUsage("all.servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
+		GuestNormalUsage("all.servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem, nil),
+		GuestNormalUsage("all.servers.last_week", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem, &lastWeek),
+		// GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestNormalUsage("all.servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
 
-		GuestPendingDeleteUsage("all.pending_delete_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem),
-		GuestPendingDeleteUsage("all.pending_delete_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
-		GuestPendingDeleteUsage("all.pending_delete_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
+		GuestPendingDeleteUsage("all.pending_delete_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem, nil),
+		GuestPendingDeleteUsage("all.pending_delete_servers.last_week", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem, &lastWeek),
+		// GuestPendingDeleteUsage("all.pending_delete_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestNormalUsage("all.servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestNormalUsage("all.servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
+
+		// GuestPendingDeleteUsage("all.pending_delete_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem),
+		// GuestPendingDeleteUsage("all.pending_delete_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestPendingDeleteUsage("all.pending_delete_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
 
 		GuestReadyUsage("all.ready_servers", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, includeSystem),
-		GuestReadyUsage("all.ready_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
-		GuestReadyUsage("all.ready_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
-		GuestRunningUsage("all.running_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
-		GuestRunningUsage("all.running_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
+		// GuestReadyUsage("all.ready_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestReadyUsage("all.ready_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
+		// GuestRunningUsage("all.running_servers.prepaid_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, includeSystem),
+		// GuestRunningUsage("all.running_servers.any_pool", rbacutils.ScopeSystem, nil, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, includeSystem),
 
 		guestRunningUsage,
 		// containerRunningUsage,
 
 		IsolatedDeviceUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv),
-		IsolatedDeviceUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
-		IsolatedDeviceUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
+		// IsolatedDeviceUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
+		// IsolatedDeviceUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
 
 		WireUsage(rangeObjs, hostTypes, providers, brands, cloudEnv),
 
@@ -308,7 +327,8 @@ func getSystemGeneralUsage(userCred mcclient.IIdentityProvider, rangeObjs []db.I
 }
 
 func getDomainGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) {
-	count := GuestNormalUsage(getKey(scope, "servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false)
+	lastWeek := time.Now().Add(-7 * 24 * time.Hour)
+	count := GuestNormalUsage(getKey(scope, "servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, nil)
 
 	var pmemTotal float64
 	var pcpuTotal float64
@@ -338,38 +358,40 @@ func getDomainGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr
 		DnsZoneUsage("domain", cred, rbacutils.ScopeDomain),
 
 		HostAllUsage("", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv),
-		HostAllUsage("prepaid_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
-		HostAllUsage("any_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
+		// HostAllUsage("prepaid_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
+		// HostAllUsage("any_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
 
 		hostEnabledUsage,
-		HostEnabledUsage("prepaid_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
-		HostEnabledUsage("any_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
+		// HostEnabledUsage("prepaid_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv),
+		// HostEnabledUsage("any_pool", cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, nil, providers, brands, cloudEnv),
 
 		BaremetalUsage(cred, rbacutils.ScopeDomain, rangeObjs, hostTypes, providers, brands, cloudEnv),
 
 		StorageUsage("", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, false, rbacutils.ScopeDomain, cred),
 		StorageUsage("system", rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, true, rbacutils.ScopeDomain, cred),
-		StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false, false, rbacutils.ScopeDomain, cred),
-		StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, false, rbacutils.ScopeDomain, cred),
-		StorageUsage("any_pool.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, true, rbacutils.ScopeDomain, cred),
-		StorageUsage("any_pool.pending_delete", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, false, rbacutils.ScopeDomain, cred),
-		StorageUsage("any_pool.pending_delete.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, true, rbacutils.ScopeDomain, cred),
+		// StorageUsage("prepaid_pool", rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false, false, rbacutils.ScopeDomain, cred),
+		// StorageUsage("any_pool", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, false, rbacutils.ScopeDomain, cred),
+		// StorageUsage("any_pool.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false, true, rbacutils.ScopeDomain, cred),
+		// StorageUsage("any_pool.pending_delete", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, false, rbacutils.ScopeDomain, cred),
+		// StorageUsage("any_pool.pending_delete.system", rangeObjs, hostTypes, nil, providers, brands, cloudEnv, true, true, rbacutils.ScopeDomain, cred),
 
-		GuestNormalUsage(getKey(scope, "servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestNormalUsage(getKey(scope, "servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		GuestNormalUsage(getKey(scope, "servers.last_week"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, &lastWeek),
+		// GuestNormalUsage(getKey(scope, "servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestNormalUsage(getKey(scope, "servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
 		guestRunningUsage,
 		// GuestRunningUsage(getKey(scope, "running_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv),
-		GuestRunningUsage(getKey(scope, "running_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestRunningUsage(getKey(scope, "running_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		// GuestRunningUsage(getKey(scope, "running_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestRunningUsage(getKey(scope, "running_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false),
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, nil),
+		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.last_week"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, &lastWeek),
+		// GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
 		GuestReadyUsage(getKey(scope, "ready_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false),
-		GuestReadyUsage(getKey(scope, "ready_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestReadyUsage(getKey(scope, "ready_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		// GuestReadyUsage(getKey(scope, "ready_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestReadyUsage(getKey(scope, "ready_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
 		NetworkUsage(getKey(scope, ""), scope, cred, providers, brands, cloudEnv, rangeObjs),
 
@@ -377,7 +399,7 @@ func getDomainGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr
 
 		BucketUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
-		nicsUsage("domain", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
+		// nicsUsage("domain", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
 
 		SnapshotUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
@@ -393,22 +415,25 @@ func getDomainGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityPr
 }
 
 func getProjectGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string) (Usage, error) {
-	count := GuestNormalUsage(getKey(scope, "servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false)
+	lastWeek := time.Now().Add(-7 * 24 * time.Hour)
+	count := GuestNormalUsage(getKey(scope, "servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, nil)
 
 	count.Include(
-		GuestNormalUsage(getKey(scope, "servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestNormalUsage(getKey(scope, "servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		GuestNormalUsage(getKey(scope, "servers.last_week"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, &lastWeek),
+		// GuestNormalUsage(getKey(scope, "servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestNormalUsage(getKey(scope, "servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 		GuestRunningUsage(getKey(scope, "running_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false),
-		GuestRunningUsage(getKey(scope, "running_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestRunningUsage(getKey(scope, "running_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		// GuestRunningUsage(getKey(scope, "running_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestRunningUsage(getKey(scope, "running_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false),
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, nil),
+		GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.last_week"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false, &lastWeek),
+		// GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestPendingDeleteUsage(getKey(scope, "pending_delete_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
 		GuestReadyUsage(getKey(scope, "ready_servers"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypeShared}, providers, brands, cloudEnv, false),
-		GuestReadyUsage(getKey(scope, "ready_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
-		GuestReadyUsage(getKey(scope, "ready_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
+		// GuestReadyUsage(getKey(scope, "ready_servers.prepaid_pool"), scope, cred, rangeObjs, hostTypes, []string{api.HostResourceTypePrepaidRecycle}, providers, brands, cloudEnv, false),
+		// GuestReadyUsage(getKey(scope, "ready_servers.any_pool"), scope, cred, rangeObjs, hostTypes, nil, providers, brands, cloudEnv, false),
 
 		NetworkUsage(getKey(scope, ""), scope, cred, providers, brands, cloudEnv, rangeObjs),
 
@@ -421,7 +446,7 @@ func getProjectGeneralUsage(scope rbacutils.TRbacScope, cred mcclient.IIdentityP
 		DisksUsage(getKey(scope, "pending_delete_disks"), rangeObjs, hostTypes, nil, providers, brands, cloudEnv, scope, cred, true, false),
 		DisksUsage(getKey(scope, "pending_delete_disks.system"), rangeObjs, hostTypes, nil, providers, brands, cloudEnv, scope, cred, true, true),
 
-		nicsUsage("", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
+		// nicsUsage("", rangeObjs, hostTypes, providers, brands, cloudEnv, scope, cred),
 
 		SnapshotUsage(scope, cred, rangeObjs, providers, brands, cloudEnv),
 
@@ -450,21 +475,24 @@ func ReportGeneralUsage(
 ) (count Usage, err error) {
 	count = make(map[string]interface{})
 
-	if scope == rbacutils.ScopeSystem || isOwner {
+	// if scope == rbacutils.ScopeSystem || isOwner {
+	if scope == rbacutils.ScopeSystem {
 		count, err = getSystemGeneralUsage(userCred, rangeObjs, hostTypes, providers, brands, cloudEnv, includeSystem)
 		if err != nil {
 			return
 		}
 	}
 
-	if scope.HigherEqual(rbacutils.ScopeDomain) {
+	// if scope.HigherEqual(rbacutils.ScopeDomain) && len(userCred.GetProjectDomainId()) > 0 {
+	if scope == rbacutils.ScopeDomain && len(userCred.GetProjectDomainId()) > 0 {
 		commonUsage, err := getDomainGeneralUsage(rbacutils.ScopeDomain, userCred, rangeObjs, hostTypes, providers, brands, cloudEnv)
 		if err == nil {
 			count.Include(commonUsage)
 		}
 	}
 
-	if scope.HigherEqual(rbacutils.ScopeProject) {
+	// if scope.HigherEqual(rbacutils.ScopeProject) && len(userCred.GetProjectId()) > 0 {
+	if scope == rbacutils.ScopeProject && len(userCred.GetProjectId()) > 0 {
 		commonUsage, err := getProjectGeneralUsage(rbacutils.ScopeProject, userCred, rangeObjs, hostTypes, providers, brands, cloudEnv)
 		if err == nil {
 			count.Include(commonUsage)
@@ -570,6 +598,10 @@ func StorageUsage(
 	count[fmt.Sprintf("%s.attached.count.owner", dPrefix)] = result.CountAttached
 	count[fmt.Sprintf("%s.detached.owner", dPrefix)] = result.DetachedCapacity
 	count[fmt.Sprintf("%s.detached.count.owner", dPrefix)] = result.CountDetached
+	count[fmt.Sprintf("%s.mounted.owner", dPrefix)] = result.AttachedCapacity
+	count[fmt.Sprintf("%s.mounted.count.owner", dPrefix)] = result.CountAttached
+	count[fmt.Sprintf("%s.unmounted.owner", dPrefix)] = result.DetachedCapacity
+	count[fmt.Sprintf("%s.unmounted.count.owner", dPrefix)] = result.CountDetached
 
 	storageCmtRate := 0.0
 	if result.Capacity > 0 {
@@ -594,6 +626,10 @@ func StorageUsage(
 	count[fmt.Sprintf("%s.attached.count", dPrefix)] = result.CountAttached
 	count[fmt.Sprintf("%s.detached", dPrefix)] = result.DetachedCapacity
 	count[fmt.Sprintf("%s.detached.count", dPrefix)] = result.CountDetached
+	count[fmt.Sprintf("%s.mounted", dPrefix)] = result.AttachedCapacity
+	count[fmt.Sprintf("%s.mounted.count", dPrefix)] = result.CountAttached
+	count[fmt.Sprintf("%s.unmounted", dPrefix)] = result.DetachedCapacity
+	count[fmt.Sprintf("%s.unmounted.count", dPrefix)] = result.CountDetached
 
 	return count
 }
@@ -621,6 +657,10 @@ func DisksUsage(
 	count[fmt.Sprintf("%s.attached.count", dPrefix)] = result.CountAttached
 	count[fmt.Sprintf("%s.detached", dPrefix)] = result.DetachedCapacity
 	count[fmt.Sprintf("%s.detached.count", dPrefix)] = result.CountDetached
+	count[fmt.Sprintf("%s.mounted", dPrefix)] = result.AttachedCapacity
+	count[fmt.Sprintf("%s.mounted.count", dPrefix)] = result.CountAttached
+	count[fmt.Sprintf("%s.unmounted", dPrefix)] = result.DetachedCapacity
+	count[fmt.Sprintf("%s.unmounted.count", dPrefix)] = result.CountDetached
 	return count
 }
 
@@ -645,7 +685,7 @@ func WireUsage(rangeObjs []db.IStandaloneModel, hostTypes []string, providers []
 	return count
 }
 
-func nicsUsage(prefix string, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage {
+/*func nicsUsage(prefix string, rangeObjs []db.IStandaloneModel, hostTypes []string, providers []string, brands []string, cloudEnv string, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) Usage {
 	count := make(map[string]interface{})
 	result := models.WireManager.TotalCount(rangeObjs, hostTypes, providers, brands, cloudEnv, scope, ownerId)
 	// including nics for pending_deleted guests
@@ -658,7 +698,7 @@ func nicsUsage(prefix string, rangeObjs []db.IStandaloneModel, hostTypes []strin
 	count[prefixKey(prefix, "nics.eip")] = result.EipNicCount
 	count[prefixKey(prefix, "nics")] = result.GuestNicCount + result.GroupNicCount + result.LbNicCount + result.DbNicCount + result.EipNicCount
 	return count
-}
+}*/
 
 func prefixKey(prefix string, key string) string {
 	if len(prefix) > 0 {
@@ -671,8 +711,15 @@ func prefixKey(prefix string, key string) string {
 func NetworkUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, providers []string, brands []string, cloudEnv string, rangeObjs []db.IStandaloneModel) Usage {
 	count := make(map[string]interface{})
 	ret := models.NetworkManager.TotalPortCount(scope, userCred, providers, brands, cloudEnv, rangeObjs)
-	count[prefixKey(prefix, "ports")] = ret.Count
-	count[prefixKey(prefix, "ports_exit")] = ret.CountExt
+	for k, v := range ret {
+		if len(k) > 0 {
+			count[prefixKey(prefix, fmt.Sprintf("ports.%s", k))] = v.Count
+			count[prefixKey(prefix, fmt.Sprintf("ports_exit.%s", k))] = v.CountExt
+		} else {
+			count[prefixKey(prefix, "ports")] = v.Count
+			count[prefixKey(prefix, "ports_exit")] = v.CountExt
+		}
+	}
 	return count
 }
 
@@ -729,26 +776,26 @@ func hostUsage(
 
 func GuestNormalUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string,
-	brands []string, cloudEnv string, includeSystem bool) Usage {
-	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, false, includeSystem)
+	brands []string, cloudEnv string, includeSystem bool, since *time.Time) Usage {
+	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, false, includeSystem, since)
 }
 
 func GuestPendingDeleteUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string,
-	brands []string, cloudEnv string, includeSystem bool) Usage {
-	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, true, includeSystem)
+	brands []string, cloudEnv string, includeSystem bool, since *time.Time) Usage {
+	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, nil, true, includeSystem, since)
 }
 
 func GuestRunningUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string,
 	brands []string, cloudEnv string, includeSystem bool) Usage {
-	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_RUNNING}, false, includeSystem)
+	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_RUNNING}, false, includeSystem, nil)
 }
 
 func GuestReadyUsage(prefix string, scope rbacutils.TRbacScope, cred mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string,
 	brands []string, cloudEnv string, includeSystem bool) Usage {
-	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_READY}, false, includeSystem)
+	return guestUsage(prefix, scope, cred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, []string{api.VM_READY}, false, includeSystem, nil)
 }
 
 func guestHypervisorsUsage(
@@ -759,11 +806,12 @@ func guestHypervisorsUsage(
 	hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string,
 	status, hypervisors []string,
 	pendingDelete, includeSystem bool,
+	since *time.Time,
 ) Usage {
 	// temporarily hide system resources
 	// XXX needs more work later
 	guest := models.GuestManager.TotalCount(scope, ownerId, rangeObjs, status, hypervisors,
-		includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv)
+		includeSystem, pendingDelete, hostTypes, resourceTypes, providers, brands, cloudEnv, since)
 	count := make(map[string]interface{})
 	count[prefix] = guest.TotalGuestCount
 	count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount
@@ -786,10 +834,10 @@ func guestHypervisorsUsage(
 
 func guestUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel,
 	hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string,
-	status []string, pendingDelete, includeSystem bool) Usage {
+	status []string, pendingDelete, includeSystem bool, since *time.Time) Usage {
 	hypervisors := sets.NewString(api.HYPERVISORS...)
 	hypervisors.Delete(api.HYPERVISOR_CONTAINER)
-	return guestHypervisorsUsage(prefix, scope, userCred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, status, hypervisors.List(), pendingDelete, includeSystem)
+	return guestHypervisorsUsage(prefix, scope, userCred, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, status, hypervisors.List(), pendingDelete, includeSystem, since)
 }
 
 /*func containerUsage(prefix string, scope rbacutils.TRbacScope, userCred mcclient.IIdentityProvider, rangeObjs []db.IStandaloneModel,

pkg/hostman/diskutils/fsutils/fsutils.go

@@ -120,7 +120,18 @@ func ResizeDiskFs(diskPath string, sizeMb int) error {
 	var cmds = []string{"parted", "-a", "none", "-s", diskPath, "--", "unit", "s", "print"}
 	lines, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
 	if err != nil {
-		log.Errorf("resize disk fs fail: %s", err)
+		hasPartTable := func() bool {
+			for _, line := range strings.Split(string(lines), "\n") {
+				if strings.Contains(line, "Partition Table") {
+					return true
+				}
+			}
+			return false
+		}
+		if hasPartTable() {
+			return nil
+		}
+		log.Errorf("resize disk fs fail, output: %s , error: %s", lines, err)
 		return err
 	}
 	parts, label := ParseDiskPartition(diskPath, lines)

pkg/hostman/diskutils/interface.go

@@ -24,7 +24,7 @@ type IDisk interface {
 	Connect() error
 	Disconnect() error
 	MountRootfs() fsdriver.IRootFsDriver
-	UmountRootfs(driver fsdriver.IRootFsDriver)
+	UmountRootfs(driver fsdriver.IRootFsDriver) error
 	ResizePartition() error
 }
 

pkg/hostman/diskutils/kvm.go

@@ -110,17 +110,18 @@ func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() fsdriver.IRootFsDriver {
 	return d.mountKvmRootfs(true)
 }
 
-func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) {
+func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) error {
 	if part := fd.GetPartition(); part != nil {
-		part.Umount()
+		return part.Umount()
 	}
+	return nil
 }
 
-func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
+func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) error {
 	if fd == nil {
-		return
+		return nil
 	}
-	d.UmountKvmRootfs(fd)
+	return d.UmountKvmRootfs(fd)
 }
 
 func (d *SKVMGuestDisk) MakePartition(fs string) error {

pkg/hostman/diskutils/vddk.go

@@ -162,11 +162,11 @@ func (vd *VDDKDisk) MountRootfs() fsdriver.IRootFsDriver {
 	return vd.kvmDisk.MountRootfs()
 }
 
-func (vd *VDDKDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
+func (vd *VDDKDisk) UmountRootfs(fd fsdriver.IRootFsDriver) error {
 	if vd.kvmDisk == nil {
-		return
+		return nil
 	}
-	vd.kvmDisk.UmountRootfs(fd)
+	return vd.kvmDisk.UmountRootfs(fd)
 }
 
 func (vd *VDDKDisk) ParsePartitions(buf string) error {
@@ -435,18 +435,18 @@ type VDDKPartition struct {
 }
 
 func (vp *VDDKPartition) Mount() bool {
-	log.Debugf("VDDKPartition.Mount not implement")
+	log.Warningf("VDDKPartition.Mount not implement")
 	return true
 }
 
 func (vp *VDDKPartition) MountPartReadOnly() bool {
-	log.Debugf("VDDKPartition.MountPartReadOnly not implement")
+	log.Warningf("VDDKPartition.MountPartReadOnly not implement")
 	return true
 }
 
-func (vp *VDDKPartition) Umount() bool {
-	log.Debugf("VDDKPartition.Umount not implement")
-	return true
+func (vp *VDDKPartition) Umount() error {
+	log.Warningf("VDDKPartition.Umount not implement")
+	return nil
 }
 
 func (vp *VDDKPartition) IsReadonly() bool {
@@ -454,7 +454,7 @@ func (vp *VDDKPartition) IsReadonly() bool {
 }
 
 func (vp *VDDKPartition) GetPhysicalPartitionType() string {
-	log.Debugf("VDDKPartition.GetPhysicalPartitionType not implement")
+	log.Warningf("VDDKPartition.GetPhysicalPartitionType not implement")
 	return ""
 }
 

pkg/hostman/guestfs/fsdriver/interface.go

@@ -50,7 +50,7 @@ type IDiskPartition interface {
 	IsMounted() bool
 	Mount() bool
 	MountPartReadOnly() bool
-	Umount() bool
+	Umount() error
 	GetMountPath() string
 	IsReadonly() bool
 	GetPhysicalPartitionType() string

pkg/hostman/guestfs/guestfishpart/guestfishpart.go

@@ -156,22 +156,25 @@ func (d *SGuestfishDiskPartition) MountPartReadOnly() bool {
 	return true
 }
 
-func (d *SGuestfishDiskPartition) Umount() bool {
-	if d.IsMounted() {
-		var tries = 0
-		for tries < 10 {
-			tries += 1
-			output, err := procutils.NewCommand("umount", d.GetMountPath()).Output()
-			if err != nil {
-				log.Errorf("failed umount %s: %s %s", d.GetMountPath(), output, err)
-				time.Sleep(time.Second * 1)
-			} else {
-				d.mounted = false
-				return true
-			}
+func (d *SGuestfishDiskPartition) Umount() error {
+	if !d.IsMounted() {
+		return nil
+	}
+
+	var tries = 0
+	var err error
+	for tries < 10 {
+		tries += 1
+		output, err := procutils.NewCommand("umount", d.GetMountPath()).Output()
+		if err != nil {
+			err = errors.Wrapf(err, "failed umount %s: %s %s", d.GetMountPath(), output, err)
+			time.Sleep(time.Second * 1)
+		} else {
+			d.mounted = false
+			return nil
 		}
 	}
-	return false
+	return err
 }
 
 func (d *SGuestfishDiskPartition) IsReadonly() bool {

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -15,6 +15,7 @@
 package kvmpart
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"os"
@@ -133,7 +134,7 @@ func (p *SKVMGuestDiskPartition) Mount() bool {
 			p.readonly = true
 		}
 	}
-	log.Infof("mount fs %s on %s success", p.fs, p.partDev)
+	log.Infof("mount fs %s on %s successfully", p.fs, p.partDev)
 	return true
 }
 
@@ -185,7 +186,7 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 			return false, errors.Wrap(err, "")
 		}
 	}
-	_, err = utils.NewFibonacciRetrierMaxTries(3, retrier).Start()
+	_, err = utils.NewFibonacciRetrierMaxTries(3, retrier).Start(context.Background())
 	if err != nil {
 		return errors.Wrap(err, "mount failed")
 	}
@@ -244,29 +245,37 @@ func (p *SKVMGuestDiskPartition) IsMounted() bool {
 	}
 }
 
-func (p *SKVMGuestDiskPartition) Umount() bool {
-	if p.IsMounted() {
-		var tries = 0
-		for tries < 10 {
-			tries += 1
-			_, err := procutils.NewCommand("umount", p.mountPath).Output()
-			if err == nil {
-				if p.fs == "xfs" {
-					uuids := fileutils2.GetDevUuid(p.partDev)
-					uuid := uuids["UUID"]
-					if len(uuid) > 0 {
-						UnlockXfsPartition(uuid)
-					}
+func (p *SKVMGuestDiskPartition) Umount() error {
+	if !p.IsMounted() {
+		return nil
+	}
+
+	var tries = 0
+	var err error
+	for tries < 10 {
+		tries += 1
+		_, err = procutils.NewCommand("umount", p.mountPath).Output()
+		if err == nil {
+			if p.fs == "xfs" {
+				uuids := fileutils2.GetDevUuid(p.partDev)
+				uuid := uuids["UUID"]
+				if len(uuid) > 0 {
+					UnlockXfsPartition(uuid)
 				}
-				procutils.NewCommand("blockdev", "--flushbufs", p.partDev).Output()
-				os.Remove(p.mountPath)
-				return true
-			} else {
-				time.Sleep(time.Second * 1)
 			}
+			if _, err := procutils.NewCommand("blockdev", "--flushbufs", p.partDev).Output(); err != nil {
+				log.Warningf("blockdev --flushbufs %s error: %v", p.partDev, err)
+			}
+			if err := os.Remove(p.mountPath); err != nil {
+				log.Warningf("remove mount path %s error: %v", p.mountPath)
+			}
+			log.Infof("umount %s successfully", p.partDev)
+			return nil
+		} else {
+			time.Sleep(time.Second * 1)
 		}
 	}
-	return false
+	return errors.Wrapf(err, "umount %s", p.mountPath)
 }
 
 func (p *SKVMGuestDiskPartition) Zerofree() {

pkg/hostman/guestfs/sshpart/sshpart.go

@@ -162,10 +162,10 @@ func (p *SSHPartition) Mount() bool {
 	return true
 }
 
-func (p *SSHPartition) Umount() bool {
+func (p *SSHPartition) Umount() error {
 	if !p.IsMounted() {
-		log.Errorf("%s is not mounted", p.mountPath)
-		return false
+		log.Warningf("%s is not mounted", p.mountPath)
+		return nil
 	}
 	var err error
 	for tries := 0; tries < 10; tries++ {
@@ -182,13 +182,12 @@ func (p *SSHPartition) Umount() bool {
 			time.Sleep(1 * time.Second)
 		} else {
 			if err := p.osRmDir(p.mountPath); err != nil {
-				log.Errorf("remove mount path %s: %v", p.mountPath, err)
-				return false
+				log.Warningf("remove mount path %s: %v", p.mountPath, err)
 			}
-			return true
+			return nil
 		}
 	}
-	return err == nil
+	return err
 }
 
 func (p *SSHPartition) IsMounted() bool {

pkg/hostman/guestman/qemu-arm.go

@@ -141,8 +141,6 @@ func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (st
 
 	if s.IsKvmSupport() {
 		cmd += "-enable-kvm"
-	} else {
-		cmd += "-no-kvm"
 	}
 
 	cmd += " -device virtio-net-pci,? 2>&1 | grep .speed= > /dev/null\n"
@@ -165,7 +163,6 @@ func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (st
 			cpuType = "max"
 		}
 	} else {
-		cmd += " -no-kvm"
 		accel = "tcg"
 		cpuType = "max"
 	}
@@ -273,7 +270,9 @@ func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (st
 		} else {
 			cmd += nicCmd
 		}
-		cmd += s.getVnicDesc(nics[i])
+		// aarch64 with addr lead to:
+		// virtio_net: probe of virtioN failed with error -22
+		cmd += s.getVnicDesc(nics[i], false)
 	}
 
 	if isolatedDevsParams != nil {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -276,7 +276,7 @@ func (s *SKVMGuestInstance) getNicAddr(index int) int {
 	return s.GetDiskAddr(pciBase + index)
 }
 
-func (s *SKVMGuestInstance) getVnicDesc(nic jsonutils.JSONObject) string {
+func (s *SKVMGuestInstance) getVnicDesc(nic jsonutils.JSONObject, withAddr bool) string {
 	bridge, _ := nic.GetString("bridge")
 	ifname, _ := nic.GetString("ifname")
 	driver, _ := nic.GetString("driver")
@@ -290,7 +290,9 @@ func (s *SKVMGuestInstance) getVnicDesc(nic jsonutils.JSONObject) string {
 	cmd += fmt.Sprintf(",netdev=%s", ifname)
 	cmd += fmt.Sprintf(",mac=%s", mac)
 
-	cmd += fmt.Sprintf(",addr=0x%x", s.getNicAddr(int(index)))
+	if withAddr {
+		cmd += fmt.Sprintf(",addr=0x%x", s.getNicAddr(int(index)))
+	}
 	if driver == "virtio" {
 		if nic.Contains("vectors") {
 			cmd += fmt.Sprintf(",vectors=%d", vectors)
@@ -312,7 +314,7 @@ func (s *SKVMGuestInstance) getQgaDesc() string {
 }
 
 func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (string, error) {
-	if s.manager.host.GetCpuArchitecture() == "aarch64" {
+	if s.manager.host.IsAarch64() {
 		return s.generateArmStartScript(data)
 	} else {
 		return s._generateStartScript(data)
@@ -617,7 +619,7 @@ function nic_mtu() {
 		} else {
 			cmd += nicCmd
 		}
-		cmd += s.getVnicDesc(nics[i])
+		cmd += s.getVnicDesc(nics[i], true)
 	}
 
 	if isolatedDevsParams != nil {

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -116,15 +116,29 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara
 		return new(deployapi.Empty), errors.Wrap(err, "disk connect failed")
 	}
 
+	unmount := func(root fsdriver.IRootFsDriver) error {
+		if err := disk.UmountRootfs(root); err != nil {
+			return errors.Wrap(err, "unmount rootfs")
+		}
+		return nil
+	}
+
 	root := disk.MountRootfs()
 	if root != nil && !root.IsResizeFsPartitionSupport() {
+		if err := unmount(root); err != nil {
+			return new(deployapi.Empty), err
+		}
 		return new(deployapi.Empty), errors.ErrNotSupported
 	}
 
 	// must umount rootfs before resize partition
-	disk.UmountRootfs(root)
-	err = disk.ResizePartition()
-	return new(deployapi.Empty), err
+	if err := unmount(root); err != nil {
+		return new(deployapi.Empty), err
+	}
+	if err := disk.ResizePartition(); err != nil {
+		return new(deployapi.Empty), errors.Wrap(err, "resize disk partition")
+	}
+	return new(deployapi.Empty), nil
 }
 
 func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) {

pkg/hostman/hostinfo/hostinfo.go

@@ -713,8 +713,9 @@ func (h *SHostInfo) detectOvsKOVersion() error {
 	}
 	lines := strings.Split(string(output), "\n")
 	for i := 0; i < len(lines); i++ {
-		if strings.HasPrefix(lines[i], "version:") {
-			log.Infof("kernel module openvswitch %s", lines[i])
+		line := lines[i]
+		if strings.HasPrefix(line, "version:") || (h.IsAarch64() && strings.HasPrefix(line, "vermagic")) {
+			log.Infof("kernel module openvswitch %s", line)
 			return nil
 		}
 	}
@@ -1721,6 +1722,14 @@ func (h *SHostInfo) GetCpuArchitecture() string {
 	return h.Cpu.CpuArchitecture
 }
 
+func (h *SHostInfo) IsAarch64() bool {
+	return h.GetCpuArchitecture() == "aarch64"
+}
+
+func (h *SHostInfo) IsX8664() bool {
+	return h.GetCpuArchitecture() == "x86_64"
+}
+
 func NewHostInfo() (*SHostInfo, error) {
 	var res = new(SHostInfo)
 	res.sysinfo = &SSysInfo{}
@@ -1731,9 +1740,9 @@ func NewHostInfo() (*SHostInfo, error) {
 		res.Cpu = cpu
 	}
 
-	if cpu.CpuArchitecture == "aarch64" {
+	if res.IsAarch64() {
 		qemutils.UseAarch64()
-	} else if cpu.CpuArchitecture != "x86_64" {
+	} else if !res.IsX8664() {
 		return nil, fmt.Errorf("unsupport cpu architecture %s", cpu.CpuArchitecture)
 	}
 

pkg/hostman/hostmetrics/hostmetrics.go

@@ -499,14 +499,14 @@ func (m *SGuestMonitor) Netio() jsonutils.JSONObject {
 		uptime, _ := host.Uptime()
 		meta.Set("uptime", jsonutils.NewInt(int64(uptime)))
 		data.Set("meta", meta)
-		data.Set("bps_sent", jsonutils.NewInt(int64(nicStat.BytesRecv*8)))
-		data.Set("bps_recv", jsonutils.NewInt(int64(nicStat.BytesSent*8)))
-		data.Set("packets_sent", jsonutils.NewInt(int64(nicStat.PacketsRecv)))
-		data.Set("packets_recv", jsonutils.NewInt(int64(nicStat.PacketsSent)))
-		data.Set("err_in", jsonutils.NewInt(int64(nicStat.Errout)))
-		data.Set("err_out", jsonutils.NewInt(int64(nicStat.Errin)))
-		data.Set("drop_in", jsonutils.NewInt(int64(nicStat.Dropout)))
-		data.Set("drop_out", jsonutils.NewInt(int64(nicStat.Dropin)))
+		data.Set("bits_recv", jsonutils.NewInt(int64(nicStat.BytesRecv*8)))
+		data.Set("bits_sent", jsonutils.NewInt(int64(nicStat.BytesSent*8)))
+		data.Set("packets_recv", jsonutils.NewInt(int64(nicStat.PacketsRecv)))
+		data.Set("packets_sent", jsonutils.NewInt(int64(nicStat.PacketsSent)))
+		data.Set("err_out", jsonutils.NewInt(int64(nicStat.Errout)))
+		data.Set("err_in", jsonutils.NewInt(int64(nicStat.Errin)))
+		data.Set("drop_out", jsonutils.NewInt(int64(nicStat.Dropout)))
+		data.Set("drop_in", jsonutils.NewInt(int64(nicStat.Dropin)))
 		res.Add(data)
 	}
 	return res

pkg/hostman/hostutils/hostutils.go

@@ -41,6 +41,7 @@ type IHost interface {
 	GetMediumType() string
 	GetMasterIp() string
 	GetCpuArchitecture() string
+	IsAarch64() bool
 	IsHugepagesEnabled() bool
 
 	IsKvmSupport() bool

pkg/hostman/storageman/disk_local.go

@@ -22,10 +22,9 @@ import (
 	"path"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -155,7 +154,9 @@ func (d *SLocalDisk) Resize(ctx context.Context, params interface{}) (jsonutils.
 		// d.Fallocate()
 	}
 
-	d.ResizeFs(d.GetPath())
+	if err := d.ResizeFs(d.GetPath()); err != nil {
+		return nil, errors.Wrapf(err, "resize fs %s", d.GetPath())
+	}
 
 	return d.GetDiskDesc(), nil
 }

pkg/hostman/storageman/disk_rbd.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -124,7 +125,10 @@ func (d *SRBDDisk) Resize(ctx context.Context, params interface{}) (jsonutils.JS
 		return nil, err
 	}
 
-	d.ResizeFs(d.GetPath())
+	if err := d.ResizeFs(d.GetPath()); err != nil {
+		return nil, errors.Wrapf(err, "resize fs %s", d.GetPath())
+	}
+
 	return d.GetDiskDesc(), nil
 }
 

pkg/hostman/storageman/diskhandlers/diskhandler.go

@@ -71,7 +71,7 @@ func AddDiskHandler(prefix string, app *appsrv.Application) {
 
 		app.AddHandler("POST",
 			fmt.Sprintf("%s/%s/<storageId>/<action>/<diskId>", prefix, keyWord),
-			auth.Authenticate(perfomrDiskActions))
+			auth.Authenticate(performDiskActions))
 		app.AddHandler("GET",
 			fmt.Sprintf("%s/%s/<storageId>/<diskId>/status", prefix, keyWord),
 			auth.Authenticate(getDiskStatus))
@@ -206,7 +206,7 @@ func saveToGlance(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	hostutils.ResponseOk(ctx, w)
 }
 
-func perfomrDiskActions(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+func performDiskActions(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	params, _, body := appsrv.FetchEnv(ctx, w, r)
 	if body == nil {
 		body = jsonutils.NewDict()
@@ -230,7 +230,8 @@ func perfomrDiskActions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	var disk storageman.IDisk
 	var err error
 
-	if action != "create" {
+	rebuild, _ := body.Bool("disk", "rebuild")
+	if action != "create" || rebuild {
 		disk, err = storage.GetDiskById(diskId)
 		if err != nil {
 			hostutils.Response(ctx, w, httperrors.NewGeneralError(errors.Wrapf(err, "GetDiskById(%s)", diskId)))

pkg/hostman/storageman/imagecachemanager_agent.go

@@ -154,7 +154,7 @@ func (c *SAgentImageCacheManager) prefetchImageCacheByUpload(ctx context.Context
 		return nil, err
 	}
 	localImgPath, _ := localImage.GetString("path")
-	localImgSize, _ := localImage.Int("size")
+	//localImgSize, _ := localImage.Int("size")
 
 	client, err := esxi.NewESXiClientFromAccessInfo(ctx, &data.Datastore)
 	if err != nil {
@@ -180,14 +180,13 @@ func (c *SAgentImageCacheManager) prefetchImageCacheByUpload(ctx context.Context
 			exists = true
 		}
 	} else {
-		ret, err := ds.CheckFile(ctx, remotePath)
+		_, err := ds.ListPath(ctx, remotePath)
 		if err != nil {
-			log.Debugf("ds.CheckFile failed: %s", err)
-		} else {
-			if int64(ret.Size) == localImgSize {
-				// exist and same size
-				exists = true
+			if errors.Cause(err) != errors.ErrNotFound {
+				return nil, errors.Wrapf(err, "unable to check file with path %s", remotePath)
 			}
+		} else {
+			exists = true
 		}
 	}
 	log.Debugf("exist: %t, remotePath: %s", exists, remotePath)

pkg/hostman/storageman/storage_agent.go

@@ -134,36 +134,36 @@ func (as *SAgentStorage) agentRebuildRoot(ctx context.Context, data jsonutils.JS
 	return vm.DoRebuildRoot(ctx, newPath, diskId)
 }
 
-func (as *SAgentStorage) agentCreateGuest(ctx context.Context, data *jsonutils.JSONDict) error {
+func (as *SAgentStorage) agentCreateGuest(ctx context.Context, data *jsonutils.JSONDict) (bool, error) {
 	hd := SHostDatastore{}
 	err := data.Unmarshal(&hd)
 	if err != nil {
-		return errors.Wrap(err, hostutils.ParamsError.Error())
+		return false, errors.Wrap(err, hostutils.ParamsError.Error())
 	}
 	host, ds, err := as.getHostAndDatastore(ctx, hd)
 	if err != nil {
-		return err
+		return false, err
 	}
 	desc, _ := data.Get("desc")
 	descDict, ok := desc.(*jsonutils.JSONDict)
 	if !ok {
-		return errors.Wrap(hostutils.ParamsError, "agentCreateGuest data format error")
+		return false, errors.Wrap(hostutils.ParamsError, "agentCreateGuest data format error")
 	}
 	createParam := esxi.SCreateVMParam{}
 	err = descDict.Unmarshal(&createParam)
 	if err != nil {
-		return errors.Wrapf(err, "%s: fail to unmarshal to esxi.SCreateVMParam", hostutils.ParamsError)
+		return false, errors.Wrapf(err, "%s: fail to unmarshal to esxi.SCreateVMParam", hostutils.ParamsError)
 	}
-	vm, err := host.CreateVM2(ctx, ds, createParam)
+	needDeploy, vm, err := host.CreateVM2(ctx, ds, createParam)
 	if err != nil {
-		return errors.Wrap(err, "SHost.CreateVM2")
+		return false, errors.Wrap(err, "SHost.CreateVM2")
 	}
 	name, _ := descDict.GetString("name")
 	err = as.tryRenameVm(ctx, vm, name)
 	if err != nil {
-		return errors.Wrapf(err, "RenameVm name '%s'", name)
+		return false, errors.Wrapf(err, "RenameVm name '%s'", name)
 	}
-	return nil
+	return needDeploy, nil
 }
 
 func (as *SAgentStorage) tryRenameVm(ctx context.Context, vm *esxi.SVirtualMachine, name string) error {
@@ -194,8 +194,12 @@ func (as *SAgentStorage) AgentDeployGuest(ctx context.Context, data interface{})
 	init := false
 	dataDict := data.(*jsonutils.JSONDict)
 	action, _ := dataDict.GetString("action")
+	var (
+		needDeploy bool
+		err        error
+	)
 	if action == "create" {
-		err := as.agentCreateGuest(ctx, dataDict)
+		needDeploy, err = as.agentCreateGuest(ctx, dataDict)
 		if err != nil {
 			return nil, errors.Wrap(err, "agentCreateGuest")
 		}
@@ -275,49 +279,52 @@ func (as *SAgentStorage) AgentDeployGuest(ctx context.Context, data interface{})
 	}
 
 	log.Debugf("host: %s, port: %d, user: %s, passwd: %s", info.Host, info.Port, info.Account, info.Password)
-	vddkInfo := deployapi.VDDKConInfo{
-		Host:   info.Host,
-		Port:   int32(info.Port),
-		User:   info.Account,
-		Passwd: info.Password,
-		Vmref:  vmref,
-	}
-	guestDesc := deployapi.GuestDesc{}
-	err = dataDict.Unmarshal(&guestDesc, "desc")
-	if err != nil {
-		return nil, errors.Wrapf(err, "%s: unmarshal to guestDesc", hostutils.ParamsError.Error())
-	}
-
-	desc, _ := dataDict.Get("desc")
-	guestDesc.Hypervisor = api.HYPERVISOR_ESXI
-	deploy, err := deployclient.GetDeployClient().DeployGuestFs(ctx, &deployapi.DeployParams{
-		DiskPath:  rootPath,
-		GuestDesc: &guestDesc,
-		DeployInfo: &deployapi.DeployInfo{
-			PublicKey:               &key,
-			Deploys:                 deployArray,
-			Password:                passwd,
-			IsInit:                  init,
-			WindowsDefaultAdminUser: true,
-		},
-		VddkInfo: &vddkInfo,
-	})
-	customize := false
-	if err != nil {
-		log.Errorf("unable to DeployGuestFs: %v", err)
-		customize = true
-	} else if deploy == nil {
-		log.Errorf("unable to DeployGuestFs: deploy is nil")
-		customize = true
-	} else if len(deploy.Os) == 0 {
-		log.Errorf("unable to DeployGuestFs: os is empty")
-		customize = true
-	}
-	if customize == true {
-		as.waitVmToolsVersion(ctx, vm)
-		err = vm.DoCustomize(ctx, desc)
+	var deploy *deployapi.DeployGuestFsResponse
+	if needDeploy {
+		vddkInfo := deployapi.VDDKConInfo{
+			Host:   info.Host,
+			Port:   int32(info.Port),
+			User:   info.Account,
+			Passwd: info.Password,
+			Vmref:  vmref,
+		}
+		guestDesc := deployapi.GuestDesc{}
+		err = dataDict.Unmarshal(&guestDesc, "desc")
 		if err != nil {
-			log.Errorf("unable to DoCustomize for vm %s: %v", vm.GetId(), err)
+			return nil, errors.Wrapf(err, "%s: unmarshal to guestDesc", hostutils.ParamsError.Error())
+		}
+
+		desc, _ := dataDict.Get("desc")
+		guestDesc.Hypervisor = api.HYPERVISOR_ESXI
+		deploy, err = deployclient.GetDeployClient().DeployGuestFs(ctx, &deployapi.DeployParams{
+			DiskPath:  rootPath,
+			GuestDesc: &guestDesc,
+			DeployInfo: &deployapi.DeployInfo{
+				PublicKey:               &key,
+				Deploys:                 deployArray,
+				Password:                passwd,
+				IsInit:                  init,
+				WindowsDefaultAdminUser: true,
+			},
+			VddkInfo: &vddkInfo,
+		})
+		customize := false
+		if err != nil {
+			log.Errorf("unable to DeployGuestFs: %v", err)
+			customize = true
+		} else if deploy == nil {
+			log.Errorf("unable to DeployGuestFs: deploy is nil")
+			customize = true
+		} else if len(deploy.Os) == 0 {
+			log.Errorf("unable to DeployGuestFs: os is empty")
+			customize = true
+		}
+		if customize == true {
+			as.waitVmToolsVersion(ctx, vm)
+			err = vm.DoCustomize(ctx, desc)
+			if err != nil {
+				log.Errorf("unable to DoCustomize for vm %s: %v", vm.GetId(), err)
+			}
 		}
 	}
 

pkg/hostman/storageman/storage_local.go

@@ -192,10 +192,13 @@ func (s *SLocalStorage) DeleteDiskfile(diskpath string) error {
 			destFile = fmt.Sprintf("%s.%d", path.Base(diskpath), time.Now().Unix())
 		)
 		if err := procutils.NewCommand("mkdir", "-p", destDir).Run(); err != nil {
+			log.Errorf("Fail to mkdir %s for recycle: %s", destDir, err)
 			return err
 		}
+		log.Infof("Move deleted disk file %s to recycle %s", diskpath, destDir)
 		return procutils.NewCommand("mv", "-f", diskpath, path.Join(destDir, destFile)).Run()
 	} else {
+		log.Infof("Delete disk file %s immediately", diskpath)
 		return procutils.NewCommand("rm", "-rf", diskpath).Run()
 	}
 }

pkg/image/models/images.go

@@ -620,7 +620,7 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		if appParams != nil && appParams.Request.ContentLength > 0 {
 			return nil, httperrors.NewInvalidStatusError("cannot upload in status %s", self.Status)
 		}
-		if minDiskSize, err := data.Int("min_disk"); err == nil {
+		if minDiskSize, err := data.Int("min_disk"); err == nil && self.DiskFormat != string(qemuimg.ISO) {
 			img, err := qemuimg.NewQemuImage(self.getLocalLocation())
 			if err != nil {
 				return nil, errors.Wrap(err, "open image")
@@ -917,9 +917,9 @@ func (manager *SImageManager) count(scope rbacutils.TRbacScope, ownerId mcclient
 		sq = sq.IsFalse("pending_deleted")
 	}
 	if isISO.IsTrue() {
-		sq = sq.Equals("disk_format", "iso")
+		sq = sq.Equals("disk_format", string(qemuimg.ISO))
 	} else if isISO.IsFalse() {
-		sq = sq.NotEquals("disk_format", "iso")
+		sq = sq.NotEquals("disk_format", string(qemuimg.ISO))
 	}
 	cnt, _ := sq.CountWithError()
 
@@ -973,13 +973,13 @@ func (manager *SImageManager) Usage(scope rbacutils.TRbacScope, ownerId mcclient
 	count := manager.count(scope, ownerId, api.IMAGE_STATUS_ACTIVE, tristate.False, false, tristate.False)
 	expandUsageCount(usages, prefix, "img", "", count)
 	count = manager.count(scope, ownerId, api.IMAGE_STATUS_ACTIVE, tristate.True, false, tristate.False)
-	expandUsageCount(usages, prefix, "iso", "", count)
+	expandUsageCount(usages, prefix, string(qemuimg.ISO), "", count)
 	count = manager.count(scope, ownerId, api.IMAGE_STATUS_ACTIVE, tristate.None, false, tristate.False)
 	expandUsageCount(usages, prefix, "imgiso", "", count)
 	count = manager.count(scope, ownerId, "", tristate.False, true, tristate.False)
 	expandUsageCount(usages, prefix, "img", "pending_delete", count)
 	count = manager.count(scope, ownerId, "", tristate.True, true, tristate.False)
-	expandUsageCount(usages, prefix, "iso", "pending_delete", count)
+	expandUsageCount(usages, prefix, string(qemuimg.ISO), "pending_delete", count)
 	count = manager.count(scope, ownerId, "", tristate.None, true, tristate.False)
 	expandUsageCount(usages, prefix, "imgiso", "pending_delete", count)
 	return usages

pkg/keystone/locale/genpolicy.go

@@ -15,7 +15,6 @@
 package locale
 
 import (
-	"fmt"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -197,7 +196,20 @@ func generatePolicies(scope rbacutils.TRbacScope, def sPolicyDefinition) []SPoli
 
 	ret := make([]SPolicyData, 0)
 	for _, role := range roleConfs {
-		name := fmt.Sprintf("%s%s%s", level, def.Name, role.name)
+		nameSegs := make([]string, 0)
+		if len(level) > 0 {
+			nameSegs = append(nameSegs, level)
+		}
+		if len(def.Name) > 0 {
+			nameSegs = append(nameSegs, def.Name)
+		}
+		if len(role.name) > 0 {
+			nameSegs = append(nameSegs, role.name)
+		}
+		name := strings.Join(nameSegs, "-")
+		if name == "sys-admin" {
+			name = "sysadmin"
+		}
 		var policy jsonutils.JSONObject
 		if def.Services != nil {
 			policy = role.policyFunc(def.Services)

pkg/keystone/locale/predefined_policies.go

@@ -437,7 +437,7 @@ var (
 			DescriptionCN: "域管理员",
 			Description:   "Domain administrator",
 			Policies: []string{
-				"domainadmin",
+				"domain-admin",
 			},
 			IsPublic: true,
 		},
@@ -446,7 +446,7 @@ var (
 			DescriptionCN: "项目主管",
 			Description:   "Project owner",
 			Policies: []string{
-				"projectadmin",
+				"project-admin",
 			},
 			IsPublic: true,
 		},
@@ -455,8 +455,8 @@ var (
 			DescriptionCN: "系统财务管理员",
 			Description:   "System finance administrator",
 			Policies: []string{
-				"sysmeteradmin",
-				"sysdashboard",
+				"sys-meter-admin",
+				"sys-dashboard",
 			},
 			IsPublic: false,
 		},
@@ -465,8 +465,8 @@ var (
 			DescriptionCN: "域财务管理员",
 			Description:   "Domain finance administrator",
 			Policies: []string{
-				"domainmeteradmin",
-				"domaindashboard",
+				"domain-meter-admin",
+				"domain-dashboard",
 			},
 			IsPublic: true,
 		},
@@ -475,8 +475,8 @@ var (
 			DescriptionCN: "项目财务管理员",
 			Description:   "Project finance administrator",
 			Policies: []string{
-				"projectmeteradmin",
-				"projectdashboard",
+				"project-meter-admin",
+				"project-dashboard",
 			},
 			IsPublic: true,
 		},
@@ -485,8 +485,8 @@ var (
 			DescriptionCN: "项目操作员",
 			Description:   "Project operator",
 			Policies: []string{
-				"projecteditor",
-				"projectdashboard",
+				"project-editor",
+				"project-dashboard",
 			},
 			IsPublic: true,
 		},
@@ -495,8 +495,8 @@ var (
 			DescriptionCN: "项目只读成员",
 			Description:   "Project read-only member",
 			Policies: []string{
-				"projectviewer",
-				"projectdashboard",
+				"project-viewer",
+				"project-dashboard",
 			},
 			IsPublic: true,
 		},

pkg/keystone/models/assignments.go

@@ -29,7 +29,6 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/keystone/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -249,9 +248,10 @@ func (manager *SAssignmentManager) ProjectAddUser(ctx context.Context, userCred
 		return err
 	}
 	if project.DomainId != user.DomainId {
-		if project.DomainId != api.DEFAULT_DOMAIN_ID && !options.Options.AllowJoinProjectsAcrossDomains {
-			return httperrors.NewInputParameterError("join user into project of default domain or identical domain")
-		} else if !db.IsAllowPerform(rbacutils.ScopeSystem, userCred, user, "join-project") {
+		// if project.DomainId != api.DEFAULT_DOMAIN_ID && !options.Options.AllowJoinProjectsAcrossDomains {
+		//	return httperrors.NewInputParameterError("join user into project of default domain or identical domain")
+		// } else
+		if !db.IsAllowPerform(rbacutils.ScopeSystem, userCred, user, "join-project") {
 			return httperrors.NewForbiddenError("not enough privilege")
 		}
 	} else {
@@ -349,9 +349,10 @@ func (manager *SAssignmentManager) projectAddGroup(ctx context.Context, userCred
 		return err
 	}
 	if project.DomainId != group.DomainId {
-		if project.DomainId != api.DEFAULT_DOMAIN_ID && !options.Options.AllowJoinProjectsAcrossDomains {
-			return httperrors.NewInputParameterError("join group into project of default domain or identical domain")
-		} else if !db.IsAllowPerform(rbacutils.ScopeSystem, userCred, group, "join-project") {
+		// if project.DomainId != api.DEFAULT_DOMAIN_ID && !options.Options.AllowJoinProjectsAcrossDomains {
+		// 	return httperrors.NewInputParameterError("join group into project of default domain or identical domain")
+		// } else
+		if !db.IsAllowPerform(rbacutils.ScopeSystem, userCred, group, "join-project") {
 			return httperrors.NewForbiddenError("not enough privilege")
 		}
 	} else {