Merge pull request #10614 from swordqiu/automated-cherry-pick-of-#10612-upstream-release-3.6

Automated cherry pick of #10612: fix(region): add usage cache, speedup response

Makefile

@@ -6,6 +6,7 @@ ROOT_DIR := $(CURDIR)
 BUILD_DIR := $(ROOT_DIR)/_output
 BIN_DIR := $(BUILD_DIR)/bin
 BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh
+DEB_BUILD_SCRIPT := $(ROOT_DIR)/build/build_deb.sh
 
 ifeq ($(ONECLOUD_CI_BUILD),)
 	GIT_COMMIT := $(shell git rev-parse --short HEAD)
@@ -64,6 +65,7 @@ endif
 
 cmdTargets:=$(filter-out cmd/host-image,$(wildcard cmd/*))
 rpmTargets:=$(foreach b,$(patsubst cmd/%,%,$(cmdTargets)),$(if $(shell [ -f "$(CURDIR)/build/$(b)/vars" ] && echo 1),rpm/$(b)))
+debTargets:=$(foreach b,$(patsubst cmd/%,%,$(cmdTargets)),$(if $(shell [ -f "$(CURDIR)/build/$(b)/vars" ] && echo 1),deb/$(b)))
 
 all: build
 
@@ -93,6 +95,9 @@ cmd/%: prepare_dir
 rpm/%: cmd/%
 	$(BUILD_SCRIPT) $*
 
+deb/%: cmd/%
+	$(DEB_BUILD_SCRIPT) $*
+
 pkg/%: prepare_dir
 	$(GO_INSTALL) $(REPO_PREFIX)/$@
 
@@ -102,6 +107,9 @@ build:
 rpm:
 	$(MAKE) $(rpmTargets)
 
+deb:
+	$(MAKE) $(debTargets)
+
 rpmclean:
 	rm -fr $(BUILD_DIR)/rpms
 
@@ -126,6 +134,11 @@ clean:
 fmt:
 	@git ls-files --exclude '*' '*.go' \
 		| grep -v '^vendor/' \
+		| while read f; do \
+			if ! grep -m1 -q '^// Code generated .* DO NOT EDIT\.$$' "$$f"; then \
+				echo "$$f"; \
+			fi ; \
+		done \
 		| xargs $(XARGS_FLAGS) gofmt -w
 
 fmt-check: fmt
@@ -198,70 +211,6 @@ mod:
 	go mod tidy
 	go mod vendor -v
 
-
-DOCKER_CENTOS_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-3
-
-define dockerCentOSBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
-docker-centos-build:
-	docker rm --force onecloud-ci-build &>/dev/null || true
-	docker run \
-		--name onecloud-docker-centos-build \
-		--rm \
-		--volume $(CURDIR):/root/onecloud \
-		--volume $(CURDIR)/_output/_cache:/root/.cache \
-		$(DOCKER_CENTOS_BUILD_IMAGE) \
-		/bin/bash -c "$$dockerCentOSBuildCmd"
-	chown -R $$(id -u):$$(id -g) _output
-	ls -lh _output/bin
-
-# NOTE we need a way to stop and remove the container started by docker-build.
-# No --tty, --stop-signal won't work
-docker-centos-build-stop:
-	docker stop --time 0 onecloud-docker-centos-build || true
-
-.PHONY: docker-centos-build
-.PHONY: docker-centos-build-stop
-
-DOCKER_ALPINE_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5
-
-define dockerAlpineBuildCmd
-set -o xtrace
-set -o errexit
-set -o pipefail
-cd /root/go/src/yunion.io/x/onecloud
-export GOFLAGS=-mod=vendor
-make $(1)
-chown -R $(shell id -u):$(shell id -g) _output
-endef
-
-docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
-docker-alpine-build:
-	docker rm --force onecloud-docker-alpine-build &>/dev/null || true
-	docker run --rm \
-		--name onecloud-docker-alpine-build \
-		-v $(CURDIR):/root/go/src/yunion.io/x/onecloud \
-		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
-		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
-		$(DOCKER_ALPINE_BUILD_IMAGE) \
-		/bin/sh -c "$$dockerAlpineBuildCmd"
-	ls -lh _output/alpine-build/bin
-
-docker-alpine-build-stop:
-	docker stop --time 0 onecloud-docker-alpine-build || true
-
-.PHONY: docker-alpine-build
-.PHONY: docker-alpine-build-stop
-
 define helpText
 Build with docker
 
@@ -321,3 +270,6 @@ image:
 
 %:
 	@:
+
+ModName:=yunion.io/x/onecloud
+include $(CURDIR)/Makefile.common.mk

Makefile.common.mk

@@ -0,0 +1,83 @@
+ifeq ($(__inc_Makefile_common_mk),)
+__inc_Makefile_common_mk:=1
+
+ifeq ($(ModName),)
+  $(error ModName must be set, e.g. yunion.io/x/onecloud)
+endif
+ModBaseName:=$(notdir $(ModName))
+
+DockerImageRegistry?=registry.cn-beijing.aliyuncs.com
+DockerImageAlpineBuild?=$(DockerImageRegistry)/yunionio/alpine-build:1.0-5
+DockerImageCentOSBuild?=$(DockerImageRegistry)/yunionio/centos-build:1.1-4
+
+
+EnvIf=$(if $($(1)),$(1)=$($(1)))
+
+define dockerCentOSBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-centos-build: export dockerCentOSBuildCmd:=$(call dockerCentOSBuildCmd,$(F))
+docker-centos-build:
+	docker rm --force docker-centos-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-centos-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/centos-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/centos-build/_cache:/root/.cache \
+		$(DockerImageCentOSBuild) \
+		/bin/bash -c "$$dockerCentOSBuildCmd"
+	ls -lh _output/centos-build/bin
+
+# NOTE we need a way to stop and remove the container started by docker-build.
+# No --tty, --stop-signal won't work
+docker-centos-build-stop:
+	docker stop --time 0 docker-centos-build-$(ModBaseName) || true
+
+.PHONY: docker-centos-build
+.PHONY: docker-centos-build-stop
+
+
+define dockerAlpineBuildCmd
+set -o xtrace
+set -o errexit
+set -o pipefail
+cd /root/go/src/yunion.io/x/$(ModBaseName)
+env \
+	$(call EnvIf,GOARCH) \
+	$(call EnvIf,GOOS) \
+	$(call EnvIf,CGO_ENABLED) \
+	make $(1)
+chown -R $(shell id -u):$(shell id -g) _output
+endef
+
+docker-alpine-build: export dockerAlpineBuildCmd:=$(call dockerAlpineBuildCmd,$(F))
+docker-alpine-build:
+	docker rm --force docker-alpine-build-$(ModBaseName) &>/dev/null || true
+	docker run \
+		--rm \
+		--name docker-alpine-build-$(ModBaseName) \
+		-v $(CURDIR):/root/go/src/yunion.io/x/$(ModBaseName) \
+		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/$(ModBaseName)/_output \
+		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
+		$(DockerImageAlpineBuild) \
+		/bin/sh -c "$$dockerAlpineBuildCmd"
+	ls -lh _output/alpine-build/bin
+
+docker-alpine-build-stop:
+	docker stop --time 0 docker-alpine-build-$(ModBaseName) || true
+
+.PHONY: docker-alpine-build
+.PHONY: docker-alpine-build-stop
+
+endif # __inc_Makefile_common_mk

build/build_deb.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$ROOT_DIR" ]; then
+	pushd $(dirname $(readlink -f "$BASH_SOURCE")) > /dev/null
+	ROOT_DIR=$(cd .. && pwd)
+	popd > /dev/null
+fi
+
+SRC_BIN=$ROOT_DIR/_output/bin
+SRC_BUILD=$ROOT_DIR/build
+OUTPUT_DIR=$ROOT_DIR/_output/debs
+
+PKG=$1
+BIN_PATH=${2:-/opt/yunion/bin}
+
+if [ -z "$PKG" ]; then
+    echo "Usage: $0 <package>"
+    exit 1
+fi
+
+BIN="$SRC_BIN/$PKG"
+ROOT="$SRC_BUILD/$PKG"
+
+if [ ! -x "$BIN" ]; then
+    echo "$BIN not exists"
+    exit 1
+fi
+
+if [ ! -x "$ROOT" ]; then
+    echo "$ROOT not exists"
+    exit 1
+fi
+
+. $ROOT/vars
+
+if [ -z "$VERSION" ]; then
+    TAG=$(git describe --abbrev=0 --tags || echo 000000)
+    VERSION=${TAG/\//-}
+    VERSION=${VERSION/v/}
+fi
+RELEASE=`date +"%y%m%d%H"`
+FULL_VERSION=$VERSION-$RELEASE
+BUILDROOT=$OUTPUT_DIR/yunion-$1-$FULL_VERSION
+rm -rf $BUILDROOT
+mkdir -p $BUILDROOT/DEBIAN
+mkdir -p $BUILDROOT/$BIN_PATH
+
+cp -rf $BIN $BUILDROOT/$BIN_PATH
+cp -rf $ROOT/root/* $BUILDROOT/
+
+
+echo "Build root ${BUILDROOT}"
+
+case $(uname -m) in
+    x86_64)
+        CURRENT_ARCH=amd64
+        ;;
+    aarch64)
+        CURRENT_ARCH=arm64
+        ;;
+esac
+
+echo "Package: yunion-$1
+Version: $FULL_VERSION
+Section: base
+Priority: optional
+Architecture: $CURRENT_ARCH
+Maintainer: wanyaoqi@yunionyun.com
+Description: Yunion $1
+" > $BUILDROOT/DEBIAN/control
+chmod 0755 $BUILDROOT/DEBIAN/control
+
+
+dpkg-deb --build $BUILDROOT

build/docker/Dockerfile.baremetal-base

@@ -1,3 +1,6 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
+RUN yum install -y https://iso.yunion.cn/3.6/rpms/packages/host/baremetal-pxerom-1.1.0-20072810.x86_64.rpm
+
 FROM frolvlad/alpine-glibc:glibc-2.28
 
 MAINTAINER "Yaoqi Wan <wanyaoqi@yunionyun.com>"
@@ -16,17 +19,17 @@ RUN mkdir -p /opt/cloud/yunion/baremetal
 RUN wget -O /opt/cloud/yunion/baremetal/pci.ids http://pci-ids.ucw.cz/v2.2/pci.ids
 
 # download baremetal-pxerom-*.x86_64.rpm and unpack rpm in current directory
-COPY ./opt/cloud/yunion/baremetal/bootia32.efi /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/bootx64.efi /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/chain.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/initramfs /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/isolinux.bin /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/kernel /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.e32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/ldlinux.e64 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/libcom32.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
-COPY ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/bootia32.efi /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/bootx64.efi /opt/cloud/yunion/baremetal
+COPY --from=build /opt/cloud/yunion/baremetal/chain.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/initramfs /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/isolinux.bin /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/kernel /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.e32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/ldlinux.e64 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/libcom32.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
+COPY --from=build ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal

build/docker/Dockerfile.host-deployer

@@ -6,4 +6,4 @@ ENV TZ Asia/Shanghai
 
 RUN mkdir -p /opt/yunion/bin
 
-ADD ./_output/alpine-build/bin/host-deployer /opt/yunion/bin/host-deployer
+ADD ./_output/centos-build/bin/host-deployer /opt/yunion/bin/host-deployer

build/docker/Dockerfile.telegraf-raid

@@ -1,3 +0,0 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
-
-ADD ./_output/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/Dockerfile.telegraf-raid-plugin

@@ -0,0 +1,3 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+
+ADD ./_output/centos-build/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/docker/multi-arch/Dockerfile.host.dockerignore

@@ -0,0 +1,2 @@
+_output/alpine-build/_cache
+

build/notify/root/opt/yunion/share/template/content@cn/VERIFY.email

@@ -47,7 +47,7 @@
 	  				</td>
 	  			</tr>
 	  			<tr>
-                    <td style="padding-bottom: 10px;">您正在验证邮箱，请在验证码输入框中输入：{{.code}}，已完成验证。</td>
+                    <td style="padding-bottom: 10px;">您正在验证邮箱，请在验证码输入框中输入：{{.code}}，以完成验证。</td>
 	  			</tr>
 	  		</table>
   		</td>

build/notify/root/opt/yunion/share/template/content@en/IMAGE_ACTIVED

@@ -0,0 +1 @@
+{{.os_type}} image {{.name}} upload completed

build/notify/root/opt/yunion/share/template/content@en/VERIFY.email

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Document</title>
+</head>
+<body>
+  <table style="width: 650px; margin-bottom: 20px;" border="0" cellpadding="0" cellspacing="0" align="center">
+  	<tr style="height: 50px; background: #333; overflow: hidden;">
+  		<td>
+  			<table style="margin-left: 20px;">
+  				<tr>
+  					<td>
+  						<img src="data:{{.login_logo_format}};base64,{{.login_logo}}" alt="" style="color: #fff; height: 32px; vertical-align: middle;">
+  					</td>
+  				</tr>
+  			</table>
+  		</td>
+  		<td>
+  			<table style="float: right;">
+  				<tr>
+  					<td style="padding-left: 20px; padding-right: 20px;">
+                        <a href="" style="text-decoration: none; color: #20A0FF; font-size: 12px; display: none;">Product</a>
+  					</td>
+  					<td style="padding-left: 20px; padding-right: 20px;">
+  						<a href="" style="text-decoration: none; color: #20A0FF; font-size: 12px; display: none;">Solution</a>
+  					</td>
+  					<td style="padding-left: 20px; padding-right: 20px;">
+  						<a href="" style="text-decoration: none; color: #20A0FF; font-size: 12px; display: none;">Service</a>
+  					</td>
+  					<td style="padding-left: 20px; padding-right: 20px;">
+  						<a href="" style="text-decoration: none; color: #20A0FF; font-size: 12px; display: none;">Help and Documentation</a>
+  					</td>
+  					<td style="padding-left: 20px; padding-right: 20px;">
+  						<a href="" style="text-decoration: none; color: #20A0FF; font-size: 12px; display: none;">About</a>
+  					</td>
+  				</tr>
+  			</table>
+  		</td>
+  	</tr>
+  	<tr>
+  		<td style="width: 100%;" colspan="2">
+  			<table style="padding: 20px 10px; width: 100%;">
+	  			<tr>
+	  				<td style="padding-bottom: 13px;">
+                        Dear {{.name}}：
+	  				</td>
+	  			</tr>
+	  			<tr>
+                    <td style="padding-bottom: 13px;">You are verifying your email, please enter the following code on the email verification page:</td>
+	  			</tr>
+	  			<tr>
+                    <td style="padding-bottom: 10px; font-size: large">{{.code}}</td>
+	  			</tr>
+	  		</table>
+  		</td>
+  	</tr>
+  	<tr style="width: 96%;">
+        <td colspan="2" style="border-top: 1px dashed #ccc; color: #333; font-size: 12px; padding-bottom: 10px; font-weight: 100;">If you are not operating by yourself, please log in to the platform in time and change your password to ensure the security of your account.</td>
+	</tr>
+	<tr>
+		<td colspan="2" style="background: #333; text-align: right; padding-right: 20px; font-size: 12px; color: #fff; height: 50px;">Copyrights © {{.copyright}}. All rights reserved.</td>
+	</tr>
+  </table>
+</body>
+</html>

build/notify/root/opt/yunion/share/template/title@en/IMAGE_ACTIVED

@@ -0,0 +1 @@
+Image {{.name}} upload completed

build/notify/root/opt/yunion/share/template/title@en/VERIFY.email

@@ -0,0 +1 @@
+{{.copyright}} Verify

cmd/climc/main.go

@@ -34,6 +34,7 @@ import (
 	_ "yunion.io/x/onecloud/cmd/climc/shell/monitor"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/notifyv2"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/servicetree"
+	_ "yunion.io/x/onecloud/cmd/climc/shell/suggestion"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/yunionconf"
 )
 

cmd/climc/shell/compute/buckets.go

@@ -452,11 +452,15 @@ func init() {
 		DomainList []string
 		// 是否允许空referer 访问
 		AllowEmptyRefer bool `help:"all empty refer access"`
+		Enabled         bool
+		RerererType     string `help:"Referer type" choices:"Black-List|White-List"`
 	}
 	R(&BucketSetRefererOption{}, "bucket-set-referer", "Set bucket referer", func(s *mcclient.ClientSession, args *BucketSetRefererOption) error {
 		conf := api.BucketRefererConf{
-			WhiteList:       args.DomainList,
+			Enabled:         args.Enabled,
 			AllowEmptyRefer: args.AllowEmptyRefer,
+			RefererType:     args.RerererType,
+			DomainList:      args.DomainList,
 		}
 		result, err := modules.Buckets.PerformAction(s, args.ID, "set-referer", jsonutils.Marshal(conf))
 		if err != nil {

cmd/climc/shell/compute/hosts.go

@@ -42,6 +42,7 @@ func init() {
 		Disabled  bool   `help:"Show disabled host only" json:"-"`
 		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud|google|ctyun"`
 		AnyMac    string `help:"Mac matches one of the host's interface"`
+		AnyIp     string `help:"IP matches one of the host's interface"`
 
 		IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"`
 
@@ -407,12 +408,14 @@ func init() {
 	})
 
 	type HostAddNetIfOptions struct {
-		ID     string `help:"ID or Name of host"`
-		WIRE   string `help:"ID or Name of wire to attach"`
-		MAC    string `help:"Mac address of NIC"`
-		INDEX  int64  `help:"nic index"`
-		Type   string `help:"Nic type" choices:"admin|ipmi"`
-		IpAddr string `help:"IP address"`
+		ID        string `help:"ID or Name of host"`
+		WIRE      string `help:"ID or Name of wire to attach"`
+		MAC       string `help:"Mac address of NIC"`
+		INDEX     int64  `help:"nic index"`
+		Type      string `help:"Nic type" choices:"admin|ipmi"`
+		IpAddr    string `help:"IP address"`
+		Bridge    string `help:"Bridge of hostwire"`
+		Interface string `help:"Interface name, eg:eth0, en0"`
 	}
 	R(&HostAddNetIfOptions{}, "host-add-netif", "Host add a NIC", func(s *mcclient.ClientSession, args *HostAddNetIfOptions) error {
 		params := jsonutils.NewDict()
@@ -426,6 +429,12 @@ func init() {
 		if len(args.IpAddr) > 0 {
 			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
+		if len(args.Bridge) > 0 {
+			params.Add(jsonutils.NewString(args.Bridge), "bridge")
+		}
+		if len(args.Interface) > 0 {
+			params.Add(jsonutils.NewString(args.Interface), "interface")
+		}
 		result, err := modules.Hosts.PerformAction(s, args.ID, "add-netif", params)
 		if err != nil {
 			return err

cmd/climc/shell/compute/secgrouprules.go

@@ -15,142 +15,17 @@
 package compute
 
 import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/cmd/climc/shell"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
-	type SecGroupRulesListOptions struct {
-		options.BaseListOptions
-		Secgroup     string   `help:"Secgroup ID or Name"`
-		SecgroupName string   `help:"Search rules by fuzzy secgroup name"`
-		Projects     []string `help:"Filter rules by project"`
-		Direction    string   `help:"filter Direction of rule" choices:"in|out"`
-		Protocol     string   `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Action       string   `help:"filter Actin of rule" choices:"allow|deny"`
-		Ports        string   `help:"filter Ports of rule"`
-		Ip           string   `help:"filter cidr of rule"`
-	}
-
-	R(&SecGroupRulesListOptions{}, "secgroup-rule-list", "List all security group", func(s *mcclient.ClientSession, args *SecGroupRulesListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.SecGroupRules.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.SecGroupRules.GetColumns(s))
-		return nil
-	})
-
-	type SecGroupRuleDetailOptions struct {
-		ID string `help:"ID or Name of security group rule"`
-	}
-	R(&SecGroupRuleDetailOptions{}, "secgroup-rule-show", "Show details of rule", func(s *mcclient.ClientSession, args *SecGroupRuleDetailOptions) error {
-		if rule, e := modules.SecGroupRules.Get(s, args.ID, nil); e != nil {
-			return e
-		} else {
-			printObject(rule)
-		}
-		return nil
-	})
-
-	R(&SecGroupRuleDetailOptions{}, "secgroup-rule-delete", "Delete a secgroup rule", func(s *mcclient.ClientSession, args *SecGroupRuleDetailOptions) error {
-		if rule, e := modules.SecGroupRules.Delete(s, args.ID, nil); e != nil {
-			return e
-		} else {
-			printObject(rule)
-		}
-		return nil
-	})
-
-	type SecGroupRulesCreateOptions struct {
-		SECGROUP  string `help:"Secgroup ID or Name" metavar:"Secgroup"`
-		Direction string `help:"Direction of rule" choices:"in|out"`
-		Action    string `help:"Action of rule" choices:"allow|deny"`
-		Protocol  string `help:"Protocol of rule" choices:"tcp|udp|icmp|any"`
-		Ports     string `help:"Ports of rule"`
-		Cidr      string `help:"Cidr of rule"`
-		Priority  int64  `help:"priority of Rule"`
-		Desc      string `help:"Description"`
-	}
-
-	R(&SecGroupRulesCreateOptions{}, "secgroup-rule-create", "Create all security group rule", func(s *mcclient.ClientSession, args *SecGroupRulesCreateOptions) error {
-		params := jsonutils.NewDict()
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
-		}
-		if args.Priority > 0 {
-			params.Add(jsonutils.NewInt(args.Priority), "priority")
-		}
-		if len(args.Direction) > 0 {
-			params.Add(jsonutils.NewString(args.Direction), "direction")
-		}
-		if len(args.Action) > 0 {
-			params.Add(jsonutils.NewString(args.Action), "action")
-		}
-		if len(args.Protocol) > 0 {
-			params.Add(jsonutils.NewString(args.Protocol), "protocol")
-		}
-		if len(args.Ports) > 0 {
-			params.Add(jsonutils.NewString(args.Ports), "ports")
-		}
-		if len(args.Cidr) > 0 {
-			params.Add(jsonutils.NewString(args.Cidr), "cidr")
-		}
-		params.Add(jsonutils.NewString(args.SECGROUP), "secgroup")
-		secgrouprules, err := modules.SecGroupRules.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(secgrouprules)
-		return nil
-	})
-
-	type SecGroupRulesUpdateOptions struct {
-		ID       string `help:"ID or name of rule"`
-		Name     string `help:"New name of rule"`
-		Priority int64  `help:"priority of Rule"`
-		Protocol string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"`
-		Ports    string `help:"Ports of rule"`
-		Cidr     string `help:"Cidr of rule"`
-		Action   string `help:"filter Actin of rule" choices:"allow|deny"`
-		Desc     string `help:"Description" metavar:"Description"`
-	}
-
-	R(&SecGroupRulesUpdateOptions{}, "secgroup-rule-update", "Update property of a security group rule", func(s *mcclient.ClientSession, args *SecGroupRulesUpdateOptions) error {
-		params := jsonutils.NewDict()
-		if len(args.Name) > 0 {
-			params.Add(jsonutils.NewString(args.Name), "name")
-		}
-		if len(args.Desc) > 0 {
-			params.Add(jsonutils.NewString(args.Desc), "description")
-		}
-		if args.Priority > 0 {
-			params.Add(jsonutils.NewInt(args.Priority), "priority")
-		}
-		if len(args.Protocol) > 0 {
-			params.Add(jsonutils.NewString(args.Protocol), "protocol")
-		}
-		if len(args.Ports) > 0 {
-			params.Add(jsonutils.NewString(args.Ports), "ports")
-		}
-		if len(args.Cidr) > 0 {
-			params.Add(jsonutils.NewString(args.Cidr), "cidr")
-		}
-		if len(args.Action) > 0 {
-			params.Add(jsonutils.NewString(args.Action), "action")
-		}
-		if rule, e := modules.SecGroupRules.Update(s, args.ID, params); e != nil {
-			return e
-		} else {
-			printObject(rule)
-		}
-		return nil
-	})
+	cmd := shell.NewResourceCmd(&modules.SecGroupRules).WithKeyword("secgroup-rule")
+	cmd.List(&compute.SecGroupRulesListOptions{})
+	cmd.Show(&options.BaseShowOptions{})
+	cmd.Delete(&options.BaseIdOptions{})
+	cmd.Create(&compute.SecGroupRulesCreateOptions{})
+	cmd.Update(&compute.SecGroupRulesUpdateOptions{})
 }

cmd/climc/shell/compute/usages.go

@@ -32,6 +32,8 @@ type GeneralUsageOptions struct {
 
 	CloudEnv string `help:"show usage of specified cloudenv" choices:"public|private|onpremise"`
 	Scope    string `help:"show usage of specified privilege scope" choices:"system|domain|project"`
+
+	Refresh bool `help:"force refresh usage statistics"`
 }
 
 func fetchHostTypeOptions(args *GeneralUsageOptions) *jsonutils.JSONDict {
@@ -48,6 +50,9 @@ func fetchHostTypeOptions(args *GeneralUsageOptions) *jsonutils.JSONDict {
 	if len(args.CloudEnv) > 0 {
 		params.Add(jsonutils.NewString(args.CloudEnv), "cloud_env")
 	}
+	if args.Refresh {
+		params.Add(jsonutils.JSONTrue, "refresh")
+	}
 	return params
 }
 

cmd/climc/shell/compute/wires.go

@@ -27,6 +27,8 @@ func init() {
 	type WireListOptions struct {
 		options.BaseListOptions
 
+		Bandwidth *int `help:"List wires by bandwidth"`
+
 		Region string `help:"List wires in region"`
 		Zone   string `help:"list wires in zone" json:"-"`
 		Vpc    string `help:"List wires in vpc"`

cmd/climc/shell/identity/identityproviders.go

@@ -32,6 +32,7 @@ import (
 func init() {
 	type IdentityProviderListOptions struct {
 		options.BaseListOptions
+		SsoDomain string `help:"Filter SSO IDP by domain" json:"sso_domain"`
 	}
 	R(&IdentityProviderListOptions{}, "idp-list", "List all identity provider", func(s *mcclient.ClientSession, args *IdentityProviderListOptions) error {
 		params, err := options.ListStructToParams(args)

cmd/climc/shell/identity/policies.go

@@ -115,7 +115,9 @@ func init() {
 		Enabled  bool   `help:"update policy enabled"`
 		Disabled bool   `help:"update policy disabled"`
 		Desc     string `help:"Description"`
-		IsSystem *bool  `help:"is_system"`
+		IsSystem bool   `help:"is_system"`
+
+		IsNotSystem bool `help:"negative is_system"`
 	}
 	updateFunc := func(s *mcclient.ClientSession, args *PolicyPatchOptions) error {
 		policyId, err := modules.Policies.GetId(s, args.ID, nil)
@@ -141,14 +143,13 @@ func init() {
 		if len(args.Desc) > 0 {
 			params.Add(jsonutils.NewString(args.Desc), "description")
 		}
-		if args.IsSystem != nil {
-			if *args.IsSystem {
-				params.Add(jsonutils.JSONTrue, "is_system")
-			} else {
-				params.Add(jsonutils.JSONFalse, "is_system")
-			}
+		if args.IsSystem {
+			params.Add(jsonutils.JSONTrue, "is_system")
 		}
-		result, err := modules.Policies.Patch(s, policyId, params)
+		if args.IsNotSystem {
+			params.Add(jsonutils.JSONFalse, "is_system")
+		}
+		result, err := modules.Policies.Update(s, policyId, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/clusters.go

@@ -167,6 +167,32 @@ func initKubeCluster() {
 		return nil
 	})
 
+	R(&o.ClusterEnableComponentCephCSIOpt{}, cmdN("component-enable-ceph-csi"), "Enable cluster ceph csi component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentCephCSIOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "enable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterEnableComponentMonitorOpt{}, cmdN("component-enable-monitor"), "Enable cluster monitor component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentMonitorOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "enable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
 	R(&o.ClusterEnableComponentFluentBitOpt{}, cmdN("component-enable-fluentbit"), "Enable cluster fluentbit component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentFluentBitOpt) error {
 		params, err := args.Params()
 		if err != nil {

cmd/climc/shell/misc/copyright.go

@@ -36,6 +36,8 @@ func init() {
 		Email     string  `help:"The Email"`
 		Docs      *string `help:"the Docs website address"`
 		License   *string `help:"the license generator website address"`
+		BrandCn   *string `help:"the brand name in Chinese"`
+		BrandEn   *string `help:"the brand name in English"`
 	}
 
 	R(&CopyrightUpdateOptions{}, "copyright-update", "update copyright", func(s *mcclient.ClientSession, args *CopyrightUpdateOptions) error {
@@ -52,6 +54,14 @@ func init() {
 			params.Add(jsonutils.NewString(*args.License), "license")
 		}
 
+		if args.BrandCn != nil {
+			params.Add(jsonutils.NewString(*args.BrandCn), "brand_cn")
+		}
+
+		if args.BrandEn != nil {
+			params.Add(jsonutils.NewString(*args.BrandEn), "brand_en")
+		}
+
 		if len(args.Copyright) > 0 {
 			params.Add(jsonutils.NewString(args.Copyright), "copyright")
 		}

cmd/climc/shell/monitor/alertrecord.go

@@ -10,4 +10,5 @@ func init() {
 	cmd := shell.NewResourceCmd(modules.AlertRecordManager)
 	cmd.List(new(options.AlertRecordListOptions))
 	cmd.Show(new(options.AlertRecordShowOptions))
+	cmd.Get("", new(options.AlertRecordTotalOptions))
 }

cmd/climc/shell/monitor/suggestsysalert.go

@@ -1,67 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package monitor
-
-import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
-	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
-)
-
-func init() {
-	aN := cmdN("suggestsysalert")
-	R(&options.SuggestSysAlertListOptions{}, aN("list"), "List all suggestsysrules",
-		func(s *mcclient.ClientSession, args *options.SuggestSysAlertListOptions) error {
-			params, err := args.Params()
-			if err != nil {
-				return err
-			}
-			if len(args.Type) > 0 {
-				params.Add(jsonutils.NewString(args.Type), "type")
-			}
-			ret, err := monitor.SuggestSysAlertManager.List(s, params)
-			if err != nil {
-				return err
-			}
-			printList(ret, monitor.SuggestSysAlertManager.GetColumns(s))
-			return nil
-		})
-
-	R(&options.SSuggestAlertShowOptions{}, aN("show"), "Show details of a alert rule",
-		func(s *mcclient.ClientSession, args *options.SSuggestAlertShowOptions) error {
-			ret, err := monitor.SuggestSysAlertManager.Get(s, args.ID, nil)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
-
-	R(&options.SuggestAlertIgnoreOptions{}, aN("ignore"), "Ignore alert result",
-		func(s *mcclient.ClientSession, args *options.SuggestAlertIgnoreOptions) error {
-			params, err := args.Params()
-			if err != nil {
-				return err
-			}
-			ret, err := monitor.SuggestSysAlertManager.PerformAction(s, args.ID, "ignore", params)
-			if err != nil {
-				return err
-			}
-			printObject(ret)
-			return nil
-		})
-}

cmd/climc/shell/notifyv2/config.go

@@ -111,7 +111,7 @@ func init() {
 	}
 	R(&ConfigGetTypesOptions{}, "notify-config-get-types", "Get all Config types", func(s *mcclient.ClientSession, args *ConfigGetTypesOptions) error {
 		param := jsonutils.Marshal(args)
-		result, err := modules.NotifyConfig.PerformClassAction(s, "get-types", param)
+		result, err := modules.NotifyReceiver.PerformClassAction(s, "get-types", param)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/suggestion/analysispredict.go

@@ -0,0 +1,12 @@
+package suggestion
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/suggestion"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(modules.AnalysisPredictManager)
+	cmd.Get("", new(options.AnalysisPredictConfigOptions))
+}

cmd/climc/shell/suggestion/suggestsysalert.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package suggestion
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(monitor.SuggestSysAlertManager)
+	cmd.List(new(options.SuggestSysAlertListOptions))
+	cmd.Show(new(options.SSuggestAlertShowOptions))
+	cmd.Perform("ignore", new(options.SuggestAlertIgnoreOptions))
+	cmd_ := shell.NewResourceCmd(monitor.SuggestSysAlertCostManager)
+	cmd_.Get("", new(options.SuggestAlertCostOptions))
+}

cmd/climc/shell/suggestion/suggestsysrule.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package monitor
+package suggestion
 
 import (
 	"yunion.io/x/onecloud/cmd/climc/shell"

go.mod

@@ -109,7 +109,7 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
-	github.com/tencentyun/cos-go-sdk-v5 v0.7.10
+	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
@@ -140,13 +140,13 @@ require (
 	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
-	yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e
+	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
 	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2
+	yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20201219153152-2d901261898c
+	yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -583,8 +583,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
-github.com/tencentyun/cos-go-sdk-v5 v0.7.10 h1:hJPLdR4RPsnubB3M9qXNBeUbcOJMVANnO/L6Att/PZU=
-github.com/tencentyun/cos-go-sdk-v5 v0.7.10/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.24/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -912,8 +912,8 @@ sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e h1:NRtqiegW4NME0v10rpChORZnJnuAUbpv31R5DsqqczQ=
-yunion.io/x/executor v0.0.0-20201201131200-44fa553abd9e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
+yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
 yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
 yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
@@ -925,13 +925,12 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 h1:gjqNa2uQzIEXyySsbvVz0Ult
 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2 h1:NeCr2J8HjcIuJvEhP0rwWA1UKP8ReOv6HVf5k9YPtyA=
-yunion.io/x/pkg v0.0.0-20201123083159-ca3aea986ff2/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c h1:J/joqbA1N2mAlOl0Uqd4LpAq3+DK5aoFMdz+p9Ld7pQ=
+yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20201219153152-2d901261898c h1:71nVDQq1oUjvZknEUNfetdiOB1jZMEfmoQlMUaoPIJs=
-yunion.io/x/sqlchemy v0.0.0-20201219153152-2d901261898c/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3 h1:C4yasW3gXronDIlIVUqHSbkA38PM1X9gxUWTYGsuIuI=
+yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/auth.go

@@ -157,6 +157,9 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 			}
 		}
 		resp.Add(domains, "domains")
+		resp.Add(jsonutils.JSONTrue, "return_full_domains")
+	} else {
+		resp.Add(jsonutils.JSONFalse, "return_full_domains")
 	}
 
 	filters := jsonutils.NewDict()
@@ -309,9 +312,20 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 	if err != nil {
 		switch httperr := err.(type) {
 		case *httputils.JSONClientError:
+			if httperr.Code >= 500 {
+				return nil, err
+			}
 			if httperr.Code == 409 || httperr.Code == 429 {
 				return nil, err
 			}
+			switch httperr.Class {
+			case "UserNotFound", "WrongPassword":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrIncorrectUsernameOrPassword, "incorrect username or password")
+			case "UserLocked":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrUserLocked, "The user has been locked, please contact the administrator")
+			case "UserDisabled":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrUserDisabled, "The user has been disabled, please contact the administrator")
+			}
 		}
 		return nil, httperrors.NewInvalidCredentialError("invalid credential")
 	}
@@ -501,34 +515,34 @@ func (h *AuthHandlers) doLogin(ctx context.Context, w http.ResponseWriter, req *
 	if body.Contains("tenantId") { // switch project
 		token, authToken, err = doTenantLogin(ctx, req, body)
 		if err != nil {
-			return errors.Wrap(err, "doTenantLogin")
+			return err
 		}
 		userInfo, err = fetchUserInfoFromToken(ctx, req, token)
 		if err != nil {
-			return errors.Wrap(err, "fetchUserInfoFromToken")
+			return err
 		}
 	} else {
 		// user/password authenticate
 		// SSO authentication
 		token, err = h.doCredentialLogin(ctx, req, body)
 		if err != nil {
-			return errors.Wrap(err, "doCredentialLogin")
+			return err
 		}
 		userInfo, err = fetchUserInfoFromToken(ctx, req, token)
 		if err != nil {
-			return errors.Wrap(err, "fetchUserInfoFromToken")
+			return err
 		}
 		s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 		isTotpInit, err := isUserTotpCredInitialed(s, token.GetUserId())
 		if err != nil {
-			return errors.Wrap(err, "isUserTotpCredInitialed")
+			return err
 		}
 		isIdpLogin := body.Contains("idp_driver")
 		authToken = clientman.NewAuthToken(token.GetTokenString(), isUserEnableTotp(userInfo), isTotpInit, isIdpLogin)
 	}
 
 	if !isUserAllowWebconsole(userInfo) {
-		return errors.Wrap(httperrors.ErrForbidden, "user forbidden login from web")
+		return httperrors.NewForbiddenError("user forbidden login from web")
 	}
 
 	saveAuthCookie(w, authToken, token)
@@ -986,6 +1000,12 @@ func getUserInfo2(s *mcclient.ClientSession, uid string, pid string, loginIp str
 		data.Add(jsonutils.JSONFalse, "non_default_domain_projects")
 	}
 
+	if options.Options.EnableQuotaCheck {
+		data.Add(jsonutils.JSONTrue, "enable_quota_check")
+	} else {
+		data.Add(jsonutils.JSONFalse, "enable_quota_check")
+	}
+
 	data.Add(jsonutils.NewString(getSsoCallbackUrl()), "sso_callback_url")
 
 	return data, nil
@@ -996,7 +1016,7 @@ func (h *AuthHandlers) getPermissionDetails(ctx context.Context, w http.Response
 
 	_, query, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 	var name string
@@ -1030,7 +1050,7 @@ func (h *AuthHandlers) doCreatePolicies(ctx context.Context, w http.ResponseWrit
 	// }
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 	s := auth.GetSession(ctx, t, FetchRegion(req), "")
@@ -1114,7 +1134,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 
@@ -1150,7 +1170,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 				return
 			}
 		}
-		httperrors.InputParameterError(ctx, w, "密码错误")
+		httperrors.InputParameterError(ctx, w, "wrong password")
 		return
 	}
 

pkg/apigateway/handler/auth_totp.go

@@ -205,7 +205,7 @@ func validateTotpRecoverySecrets(s *mcclient.ClientSession, uid string, question
 func initTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	t, authToken, err := fetchAuthInfo(ctx, req)
 	if err != nil {
-		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail %s", err)
+		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail: %s", err)
 		return
 	}
 	if authToken.IsTotpInitialized() {
@@ -232,14 +232,14 @@ func initTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Reque
 func validatePasscodeHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	t, authToken, err := fetchAuthInfo(ctx, req)
 	if err != nil {
-		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail %s", err)
+		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail: %s", err)
 		return
 	}
 
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 
@@ -271,14 +271,14 @@ func validatePasscodeHandler(ctx context.Context, w http.ResponseWriter, req *ht
 func resetTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	t, _, err := fetchAuthInfo(ctx, req)
 	if err != nil {
-		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail %s", err)
+		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail: %s", err)
 		return
 	}
 
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 
@@ -309,7 +309,7 @@ func resetTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Requ
 func listTotpRecoveryQuestions(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	t, _, err := fetchAuthInfo(ctx, req)
 	if err != nil {
-		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail %s", err)
+		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail: %s", err)
 		return
 	}
 
@@ -335,14 +335,14 @@ func listTotpRecoveryQuestions(ctx context.Context, w http.ResponseWriter, req *
 func resetTotpRecoveryQuestions(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	t, _, err := fetchAuthInfo(ctx, req)
 	if err != nil {
-		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail %s", err)
+		httperrors.InvalidCredentialError(ctx, w, "fetchAuthInfo fail: %s", err)
 		return
 	}
 
 	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
-		httperrors.InvalidInputError(ctx, w, "body is empty")
+		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
 

pkg/apigateway/handler/idp.go

@@ -170,7 +170,7 @@ func (h *AuthHandlers) handleSsoLogin(ctx context.Context, w http.ResponseWriter
 	case "POST":
 		formData, err := appsrv.Fetch(req)
 		if err != nil {
-			httperrors.InputParameterError(ctx, w, "fetch formdata error: %s", err)
+			httperrors.InputParameterError(ctx, w, "fetch form data error: %s", err)
 		}
 		body, err = jsonutils.ParseQueryString(string(formData))
 		if err != nil {

pkg/apigateway/handler/misc.go

@@ -42,6 +42,8 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
+const contentTypeSpreadsheet = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
 const (
 	HOST_MAC                    = "*MAC地址"
 	HOST_NAME                   = "*名称"
@@ -154,8 +156,8 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 
 	fileHeader := hostfiles[0].Header
 	contentType := fileHeader.Get("Content-Type")
-	if contentType != "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
-		e := httperrors.NewInputParameterError("Wrong content type %s, required application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", contentType)
+	if contentType != contentTypeSpreadsheet {
+		e := httperrors.NewInputParameterError("Wrong content type %s, want %s", contentType, contentTypeSpreadsheet)
 		httperrors.JsonClientError(ctx, w, e)
 		return
 	}
@@ -199,7 +201,7 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	}
 
 	if !titlesOk {
-		httperrors.InputParameterError(ctx, w, "template file is invalid.please check.")
+		httperrors.InputParameterError(ctx, w, "template file is invalid. please check.")
 		return
 	}
 
@@ -302,8 +304,8 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 
 	fileHeader := userfiles[0].Header
 	contentType := fileHeader.Get("Content-Type")
-	if contentType != "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" {
-		e := httperrors.NewInputParameterError("Wrong content type %s, required application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", contentType)
+	if contentType != contentTypeSpreadsheet {
+		e := httperrors.NewInputParameterError("Wrong content type %s, want %s", contentType, contentTypeSpreadsheet)
 		httperrors.JsonClientError(ctx, w, e)
 		return
 	}
@@ -328,7 +330,7 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 	// skipped header row
 	rows := xlsx.GetRows("users")
 	if len(rows) <= 1 {
-		e := httperrors.NewInputParameterError("empty file")
+		e := httperrors.NewInputParameterError("empty file content")
 		httperrors.JsonClientError(ctx, w, e)
 		return
 	} else if len(rows) > BATCH_USER_REGISTER_QUANTITY_LIMITATION {
@@ -431,7 +433,7 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW
 	params := appctx.AppContextParams(ctx)
 	template, ok := params["<template_id>"]
 	if !ok || len(template) == 0 {
-		httperrors.InvalidInputError(ctx, w, "not found")
+		httperrors.InvalidInputError(ctx, w, "template_id")
 		return
 	}
 

pkg/apigateway/handler/resource.go

@@ -142,11 +142,11 @@ func fetchIdList(ctx context.Context, query jsonutils.JSONObject, w http.Respons
 	if e == nil && len(idlist) > 0 {
 		queryDict := query.(*jsonutils.JSONDict)
 		queryDict.Remove("id")
-		log.Debugf("Get idlist: %s", idlist)
+		log.Debugf("Get id list: %s", idlist)
 		return jsonutils.JSONArray2StringArray(idlist)
 	} else {
-		log.Debugf("Cannot find idlist in query: %s", query)
-		httperrors.InvalidInputError(ctx, w, "No idlist found")
+		log.Debugf("Cannot find id list in query: %s", query)
+		httperrors.InvalidInputError(ctx, w, "No id list found")
 		return nil
 	}
 }

pkg/apis/cloudevent/cloudevent.go

@@ -23,7 +23,7 @@ import (
 
 type CloudeventListInput struct {
 	apis.ModelBaseListInput
-	apis.ProjectizedResourceListInput
+	apis.DomainizedResourceListInput
 
 	compute.CloudenvResourceListInput
 
@@ -47,6 +47,6 @@ type CloudeventListInput struct {
 
 type CloudeventDetails struct {
 	apis.ModelBaseDetails
-	apis.ProjectizedResourceInfo
+	apis.DomainizedResourceInfo
 	SCloudevent
 }

pkg/apis/cloudid/cloudrole.go

@@ -27,6 +27,7 @@ type CloudroleListInput struct {
 	apis.StatusInfrasResourceBaseListInput
 
 	CloudaccountResourceListInput
+	CloudgroupResourceListInput
 }
 
 type CloudroleDetails struct {

pkg/apis/compute/bucket.go

@@ -320,12 +320,16 @@ func (input *BucketCORSRules) Validate() error {
 }
 
 type BucketRefererConf struct {
-	// 白名单域名列表
-	WhiteList []string
-	// 黑名单域名列表
-	BlackList []string
+	// Referer Type
+	// enmu: Black-List, White-List
+	RefererType string
+	// 域名列表
+	DomainList []string
 	// 是否允许空referer 访问
 	AllowEmptyRefer bool
+
+	// 是否开启
+	Enabled bool `json:"enabled"`
 }
 
 func (input *BucketRefererConf) Validate() error {

pkg/apis/compute/cloudaccount.go

@@ -335,6 +335,15 @@ type CloudaccountPerformPublicInput struct {
 type CloudaccountPerformPrepareNetsInput struct {
 	CloudaccountCreateInput
 
+	// description: Recommended network subnet mask
+	// example: 24
+	NetworkBits *int8 `json:"network_bits"`
+	// description: The upper limit of host number
+	// example: 254
+	HostNumberUpperLimit *int `json:"host_number_upper_limit"`
+	// description: The lower limit of host number
+	// example: 2
+	HostNumberLowerLimit *int `json:"host_number_lower_limit"`
 	// enum: vcenter,datacenter,cluster
 	WireLevelForVmware string `json:"wire_level_for_vmware"`
 }
@@ -427,3 +436,9 @@ type GetCloudaccountSamlOutput struct {
 	// initial SAML SSO login URL for this cloudaccount
 	InitLoginUrl string `json:"init_login_url,allowempty"`
 }
+
+type CloudaccountEnableAutoSyncInput struct {
+	// 云账号状态必须是connected
+	// 最小值为region服务的minimal_sync_interval_seconds
+	SyncIntervalSeconds int `json:"sync_interval_seconds"`
+}

pkg/apis/compute/cloudaccount_const.go

@@ -101,6 +101,50 @@ var (
 		CLOUD_PROVIDER_GOOGLE,
 		CLOUD_PROVIDER_CTYUN,
 	}
+
+	CLOUD_PROVIDER_HOST_TYPE_MAP = map[string][]string{
+		CLOUD_PROVIDER_ONECLOUD: []string{
+			HOST_TYPE_KVM,
+			HOST_TYPE_BAREMETAL,
+			HOST_TYPE_HYPERVISOR,
+		},
+		CLOUD_PROVIDER_VMWARE: []string{
+			HOST_TYPE_ESXI,
+		},
+		CLOUD_PROVIDER_ALIYUN: []string{
+			HOST_TYPE_ALIYUN,
+		},
+		CLOUD_PROVIDER_APSARA: []string{
+			HOST_TYPE_APSARA,
+		},
+		CLOUD_PROVIDER_QCLOUD: []string{
+			HOST_TYPE_QCLOUD,
+		},
+		CLOUD_PROVIDER_AZURE: []string{
+			HOST_TYPE_AZURE,
+		},
+		CLOUD_PROVIDER_AWS: []string{
+			HOST_TYPE_AWS,
+		},
+		CLOUD_PROVIDER_HUAWEI: []string{
+			HOST_TYPE_HUAWEI,
+		},
+		CLOUD_PROVIDER_OPENSTACK: []string{
+			HOST_TYPE_OPENSTACK,
+		},
+		CLOUD_PROVIDER_UCLOUD: []string{
+			HOST_TYPE_UCLOUD,
+		},
+		CLOUD_PROVIDER_ZSTACK: []string{
+			HOST_TYPE_ZSTACK,
+		},
+		CLOUD_PROVIDER_GOOGLE: []string{
+			HOST_TYPE_GOOGLE,
+		},
+		CLOUD_PROVIDER_CTYUN: []string{
+			HOST_TYPE_CTYUN,
+		},
+	}
 )
 
 const (

pkg/apis/compute/cloudregions_const.go

@@ -114,6 +114,7 @@ const (
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
 	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
 	CITY_LAS_VEGAS      = "Las Vegas"      //拉斯维加斯
+	CITY_PHOENIX        = "Phoenix"        //菲尼克斯
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -150,7 +151,8 @@ const (
 	CITY_STOCKHOLM = "Stockholm" //斯德哥尔摩
 
 	// 巴西
-	CITY_SAO_PAULO = "Sao Paulo" //圣保罗
+	CITY_SAO_PAULO      = "Sao Paulo"      //圣保罗
+	CITY_RIO_DE_JANEIRO = "Rio de Janeiro" // 里约热内卢
 
 	// 智利
 	CITY_SANTIAGO = "Santiago" // 圣地亚哥
@@ -175,6 +177,9 @@ const (
 	// 尼日利亚
 	CITY_LAGOS = "Lagos" //拉哥斯
 
+	// 巴林王国 (中东国家)
+	CITY_BAHRAIN = "Bahrain" // 巴林
+
 	// 越南
 	CITY_HO_CHI_MINH = "Ho Chi Minh" //???
 
@@ -205,4 +210,5 @@ const (
 	COUNTRY_CODE_NO = "NO" //挪威
 	COUNTRY_CODE_MX = "MX" //墨西哥
 	COUNTRY_CODE_CL = "CL" //智利
+	COUNTRY_CODE_BH = "BH" //巴林
 )

pkg/apis/compute/dbinstance.go

@@ -166,11 +166,7 @@ type SDBInstanceChangeConfigInput struct {
 	apis.Meta
 
 	InstanceType string
-	VCpuCount    int
-	VmemSizeMb   int
-	StorageType  string
 	DiskSizeGB   int
-	Category     string
 }
 
 type SDBInstanceRecoveryConfigInput struct {
@@ -294,6 +290,8 @@ type DBInstanceDetails struct {
 
 	SDBInstance
 
+	Secgroups []apis.StandaloneShortDesc `json:"secgroups"`
+
 	// 安全组名称
 	// example: Default
 	Secgroup string `json:"secgroup"`

pkg/apis/compute/disk.go

@@ -106,6 +106,7 @@ type DiskListInput struct {
 	apis.VirtualResourceListInput
 	apis.ExternalizedResourceBaseListInput
 	apis.MultiArchResourceBaseListInput
+	apis.AutoDeleteResourceBaseListInput
 	billing.BillingResourceListInput
 	StorageFilterListInput
 
@@ -134,8 +135,6 @@ type DiskListInput struct {
 
 	DiskSize int `json:"disk_size"`
 
-	AutoDelete *bool `json:"auto_delete"`
-
 	FsFormat string `json:"fs_format"`
 
 	// 镜像

pkg/apis/compute/elasticcache.go

@@ -159,6 +159,12 @@ type ElasticcacheCreateInput struct {
 	// 主可用区名称或Id
 	Zone string `json:"zone"`
 
+	// 备可用区名称或Id列表
+	// 默认副本与主可用区一致
+	// 支持此参数的云厂商: 腾讯云
+	// required: false
+	SlaveZones []string `json:"slave_zones"`
+
 	// Ip子网名称或Id,建议使用Id
 	// required: true
 	Network string `json:"network"`

pkg/apis/compute/guests.go

@@ -372,7 +372,10 @@ type GuestMigrateInput struct {
 }
 
 type GuestLiveMigrateInput struct {
+	// 指定期望的迁移目标宿主机
 	PreferHost string `json:"prefer_host"`
+	// 是否跳过CPU检查，默认要做CPU检查
+	SkipCpuCheck *bool `json:"skip_cpu_check"`
 }
 
 type GuestSetSecgroupInput struct {
@@ -457,3 +460,25 @@ type ServerStopInput struct {
 	// 目前仅阿里云，腾讯云此参数生效
 	StopCharging bool `json:"stop_charging"`
 }
+
+type ServerDeleteInput struct {
+	// 是否越过回收站直接删除
+	// default: false
+	OverridePendingDelete bool
+
+	// 是否仅删除本地资源
+	// default: false
+	Purge bool
+
+	// 是否删除快照
+	// default: false
+	DeleteSnapshots bool
+
+	// 是否删除关联的EIP
+	// default: false
+	DeleteEip bool
+
+	// 是否删除关联的数据盘
+	// default: false
+	DeleteDisks bool
+}

pkg/apis/compute/host.go

@@ -72,6 +72,8 @@ type HostListInput struct {
 	ResourceType string `json:"resource_type"`
 	// filter by mac of any network interface
 	AnyMac string `json:"any_mac"`
+	// filter by ip of any network interface
+	AnyIp string `json:"any_ip"`
 	// filter storages not attached to this host
 	StorageNotAttached *bool `json:"storage_not_attached"`
 	// filter by Hypervisor

pkg/apis/compute/schedtag.go

@@ -59,6 +59,7 @@ type SchedtagFilterListInput struct {
 type SchedtagListInput struct {
 	apis.StandaloneResourceListInput
 	apis.ScopedResourceBaseListInput
+	CloudproviderResourceInput
 
 	// fitler by resource_type
 	ResourceType []string `json:"resource_type"`

pkg/apis/compute/scheduled_task.go

@@ -23,6 +23,9 @@ import (
 type ScheduledTaskDetails struct {
 	apis.VirtualResourceDetails
 	SScheduledTask
+
+	// 描述
+	TimerDesc string `json:"timer_desc"`
 	// 定时方式触发
 	Timer TimerDetails `json:"timer"`
 	// 周期方式触发

pkg/apis/compute/secgroup.go

@@ -24,14 +24,12 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 )
 
-type SSecgroupRuleCreateInput struct {
-	apis.ResourceBaseCreateInput
-
+type SSecgroupRuleResource struct {
 	// 优先级, 数字越大优先级越高
 	// minimum: 1
 	// maximum: 100
 	// required: true
-	Priority int `json:"priority"`
+	Priority *int `json:"priority"`
 
 	// 协议
 	// required: true
@@ -68,7 +66,7 @@ type SSecgroupRuleCreateInput struct {
 	// required: true
 	Direction string `json:"direction"`
 
-	// ip或cidr地址
+	// ip或cidr地址, 若指定peer_secgroup_id此参数不生效
 	// example: 192.168.222.121
 	CIDR string `json:"cidr"`
 
@@ -84,17 +82,36 @@ type SSecgroupRuleCreateInput struct {
 	// example: test to create rule
 	Description string `json:"description"`
 
-	// 仅单独创建安全组规则时需要指定安全组
-	// required: true
-	Secgroup string `json:"secgroup"`
-
-	// swagger:ignore
-	SecgroupId string
+	// 对端安全组Id, 此参数和cidr参数互斥，并且优先级高于cidr, 同事peer_secgroup_id不能和它所在的安全组ID相同
+	// required: false
+	PeerSecgroupId string `json:"peer_secgroup_id"`
 }
 
-func (input *SSecgroupRuleCreateInput) Check() error {
+type SSecgroupRuleCreateInput struct {
+	apis.ResourceBaseCreateInput
+	SSecgroupRuleResource
+
+	// swagger:ignore
+	Secgroup string `json:"secgroup"  yunion-deprecated-by:"secgroup_id"`
+
+	// 安全组ID
+	// required: true
+	SecgroupId string `json:"secgroup_id"`
+}
+
+type SSecgroupRuleUpdateInput struct {
+	apis.ResourceBaseUpdateInput
+
+	SSecgroupRuleResource
+}
+
+func (input *SSecgroupRuleResource) Check() error {
+	priority := 1
+	if input.Priority != nil {
+		priority = *input.Priority
+	}
 	rule := secrules.SecurityRule{
-		Priority:  input.Priority,
+		Priority:  priority,
 		Direction: secrules.TSecurityRuleDirection(input.Direction),
 		Action:    secrules.TSecurityRuleAction(input.Action),
 		Protocol:  input.Protocol,

pkg/apis/compute/secgrouprule.go

@@ -22,5 +22,6 @@ type SecgroupRuleDetails struct {
 	SSecurityGroupRule
 	SecurityGroupResourceInfo
 
-	ProjectId string `json:"tenant_id"`
+	ProjectId    string `json:"tenant_id"`
+	PeerSecgroup string `json:"peer_secgroup"`
 }

pkg/apis/compute/snapshot_const.go

@@ -19,12 +19,13 @@ const (
 	SNAPSHOT_MANUAL = "manual"
 	SNAPSHOT_AUTO   = "auto"
 
-	SNAPSHOT_CREATING    = "creating"
-	SNAPSHOT_ROLLBACKING = "rollbacking"
-	SNAPSHOT_FAILED      = "create_failed"
-	SNAPSHOT_READY       = "ready"
-	SNAPSHOT_DELETING    = "deleting"
-	SNAPSHOT_UNKNOWN     = "unknown"
+	SNAPSHOT_CREATING      = "creating"
+	SNAPSHOT_ROLLBACKING   = "rollbacking"
+	SNAPSHOT_FAILED        = "create_failed"
+	SNAPSHOT_READY         = "ready"
+	SNAPSHOT_DELETE_FAILED = "delete_failed"
+	SNAPSHOT_DELETING      = "deleting"
+	SNAPSHOT_UNKNOWN       = "unknown"
 
 	SNAPSHOT_POLICY_CREATING = "creating"
 

pkg/apis/compute/storage_const.go

@@ -146,6 +146,25 @@ var (
 	SHARED_STORAGE = []string{STORAGE_NFS, STORAGE_GPFS, STORAGE_RBD}
 )
 
+func IsDiskTypeMatch(t1, t2 string) bool {
+	switch t1 {
+	case DISK_TYPE_ROTATE:
+		if t2 == DISK_TYPE_SSD {
+			return false
+		} else {
+			return true
+		}
+	case DISK_TYPE_SSD:
+		if t2 == DISK_TYPE_ROTATE {
+			return false
+		} else {
+			return true
+		}
+	default:
+		return true
+	}
+}
+
 type StorageResourceInput struct {
 	// 存储（ID或Name）
 	StorageId string `json:"storage_id"`

pkg/apis/compute/vpcs_const.go

@@ -33,6 +33,8 @@ const (
 
 	DEFAULT_VPC_ID = "default"
 	NORMAL_VPC_ID  = "normal" // 没有关联VPC的安全组，统一使用normal
+
+	CLASSIC_VPC_NAME = "-"
 )
 
 type UsableResourceListInput struct {

pkg/apis/compute/wire.go

@@ -87,4 +87,6 @@ type WireListInput struct {
 	ZonalFilterListBase
 
 	HostResourceInput
+
+	Bandwidth *int `json:"bandwidth"`
 }

pkg/apis/compute/zz_generated.model.go

@@ -2068,7 +2068,6 @@ type SScheduledTask struct {
 	apis.SEnabledResourceBase
 	ScheduledType string `json:"scheduled_type"`
 	STimer
-	TimerDesc    string `json:"timer_desc"`
 	ResourceType string `json:"resource_type"`
 	Operation    string `json:"operation"`
 	LabelType    string `json:"label_type"`

pkg/apis/identity/usrext.go

@@ -31,11 +31,12 @@ type SUserExtended struct {
 	Email       string
 	Mobile      string
 
-	LocalId       int
-	LocalName     string
-	DomainName    string
-	DomainEnabled bool
-	IsLocal       bool
+	LocalId              int
+	LocalName            string
+	LocalFailedAuthCount int
+	DomainName           string
+	DomainEnabled        bool
+	IsLocal              bool
 	// IdpId         string
 	// IdpName       string
 }

pkg/apis/input.go

@@ -205,6 +205,8 @@ type PerformStatusInput struct {
 	// 更改的目标状态值
 	// required:true
 	Status string `json:"status"`
+	// swagger:ignore
+	BlockJobsCount int `json:"block_jobs_count"`
 
 	// 更改状态的原因描述
 	// required:false

pkg/apis/list.go

@@ -319,3 +319,7 @@ type EnabledStatusInfrasResourceBaseListInput struct {
 type MultiArchResourceBaseListInput struct {
 	OsArch string `json:"os_arch"`
 }
+
+type AutoDeleteResourceBaseListInput struct {
+	AutoDelete *bool
+}

pkg/apis/logger/input.go

@@ -27,4 +27,14 @@ type BaremetalEventListInput struct {
 	Since time.Time `json:"since"`
 	// until
 	Until time.Time `json:"until"`
+	// host_id
+	HostId []string `json:"host_id"`
+	// id
+	Id []int64 `json:"id"`
+	// EventId
+	EventId []string `json:"event_id"`
+	// Type
+	Type []string `json:"type"`
+	// ipmi_ip
+	IpmiIp []string `json:"ipmi_ip"`
 }

pkg/apis/monitor/alertquery_const.go

@@ -0,0 +1,147 @@
+package monitor
+
+var (
+	ServerTags = map[string]string{
+		"host":             "host",
+		"host_id":          "host_id",
+		"vm_id":            "id",
+		"vm_ip":            "ips",
+		"vm_name":          "name",
+		"zone":             "zone",
+		"zone_id":          "zone_id",
+		"zone_ext_id":      "zone_ext_id",
+		"os_type":          "os_type",
+		"status":           "status",
+		"cloudregion":      "cloudregion",
+		"cloudregion_id":   "cloudregion_id",
+		"region_ext_id":    "region_ext_id",
+		"tenant":           "tenant",
+		"tenant_id":        "tenant_id",
+		"brand":            "brand",
+		"scaling_group_id": "vm_scaling_group_id",
+		"domain_id":        "domain_id",
+		"project_domain":   "project_domain",
+	}
+
+	HostTags = map[string]string{
+		"host_id":        "id",
+		"host_ip":        "ips",
+		"host":           "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	RdsTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"rds_id":         "id",
+		"rds_ip":         "ips",
+		"rds_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	RedisTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"redis_id":       "id",
+		"redis_ip":       "ips",
+		"redis_name":     "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	OssTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"oss_id":         "id",
+		"oss_ip":         "ips",
+		"oss_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"region_ext_id":  "region_ext_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	ElbTags = map[string]string{
+		"host":           "host",
+		"host_id":        "host_id",
+		"elb_id":         "id",
+		"elb_ip":         "ips",
+		"elb_name":       "name",
+		"zone":           "zone",
+		"zone_id":        "zone_id",
+		"zone_ext_id":    "zone_ext_id",
+		"os_type":        "os_type",
+		"status":         "status",
+		"region":         "region",
+		"cloudregion":    "cloudregion",
+		"cloudregion_id": "cloudregion_id",
+		"tenant":         "tenant",
+		"tenant_id":      "tenant_id",
+		"brand":          "brand",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+
+	CloudAccountTags = map[string]string{
+		"cloudaccount_id":   "id",
+		"cloudaccount_name": "name",
+		"brand":             "brand",
+		"domain_id":         "domain_id",
+		"project_domain":    "project_domain",
+	}
+
+	TenantTags = map[string]string{
+		"tenant_id":      "id",
+		"tenant":         "name",
+		"domain_id":      "domain_id",
+		"project_domain": "project_domain",
+	}
+	DomainTags = map[string]string{
+		"domain_id":      "id",
+		"project_domain": "name",
+	}
+)

pkg/apis/monitor/alertresource.go

@@ -22,7 +22,7 @@ type AlertResourceType string
 
 const (
 	// AlertResourceTypeNode means onecloud system infrastructure controller or host node
-	AlertResourceTypeNode AlertResourceType = "node"
+	AlertResourceTypeNode AlertResourceType = "host"
 	// AlertResourceTypeCloudaccount means cloudaccount resource
 	AlertResourceTypeCloudaccount AlertResourceType = "cloudaccount"
 	// AlertResourceTypeVM means virtual machine guest resource

pkg/apis/monitor/commalert.go

@@ -120,10 +120,11 @@ type CommonAlertDetails struct {
 }
 
 type CommonAlertMetricDetails struct {
-	Comparator    string  `json:"comparator"`
-	Threshold     float64 `json:"threshold"`
-	ConditionType string  `json:"condition_type"`
-	ThresholdStr  string  `json:"threshold_str"`
+	Comparator    string    `json:"comparator"`
+	Threshold     float64   `json:"threshold"`
+	WithinRange   []float64 `json:"within_range"`
+	ConditionType string    `json:"condition_type"`
+	ThresholdStr  string    `json:"threshold_str"`
 	// metric points'value的运算方式
 	Reduce                 string           `json:"reduce"`
 	DB                     string           `json:"db"`

pkg/apis/monitor/metric.go

@@ -9,6 +9,8 @@ const (
 	METRIC_RES_TYPE_OSS          = "oss"
 	METRIC_RES_TYPE_RDS          = "rds"
 	METRIC_RES_TYPE_CLOUDACCOUNT = "cloudaccount"
+	METRIC_RES_TYPE_TENANT       = "tenant"
+	METRIC_RES_TYPE_DOMAIN       = "domain"
 
 	METRIC_UNIT_PERCENT = "%"
 	METRIC_UNIT_BPS     = "bps"
@@ -30,8 +32,8 @@ var (
 	MetricUnit = []string{METRIC_UNIT_PERCENT, METRIC_UNIT_BPS, METRIC_UNIT_MBPS, METRIC_UNIT_BYTEPS, "count/s",
 		METRIC_UNIT_COUNT, METRIC_UNIT_MS, METRIC_UNIT_BYTE, METRIC_UNIT_RMB}
 	ResTypeScoreMap = map[string]int{
-		METRIC_RES_TYPE_HOST:         1,
-		METRIC_RES_TYPE_GUEST:        2,
+		METRIC_RES_TYPE_GUEST:        1,
+		METRIC_RES_TYPE_HOST:         2,
 		METRIC_RES_TYPE_OSS:          3,
 		METRIC_RES_TYPE_RDS:          4,
 		METRIC_RES_TYPE_REDIS:        5,

pkg/apis/monitor/unifiedmonitor_const.go

@@ -21,12 +21,14 @@ var (
 		"cloudaccount": "cloudaccount_name",
 	}
 	MEASUREMENT_TAG_ID = map[string]string{
-		"host":         "host_id",
-		"guest":        "vm_id",
-		"redis":        "redis_id",
-		"rds":          "rds_id",
-		"oss":          "oss_id",
-		"cloudaccount": "cloudaccount_id",
+		METRIC_RES_TYPE_HOST:         "host_id",
+		METRIC_RES_TYPE_GUEST:        "vm_id",
+		METRIC_RES_TYPE_REDIS:        "redis_id",
+		METRIC_RES_TYPE_RDS:          "rds_id",
+		METRIC_RES_TYPE_OSS:          "oss_id",
+		METRIC_RES_TYPE_CLOUDACCOUNT: "cloudaccount_id",
+		METRIC_RES_TYPE_TENANT:       "tenant_id",
+		METRIC_RES_TYPE_DOMAIN:       "domain_id",
 	}
 	AlertReduceFunc = map[string]string{
 		"avg":          "average value",

pkg/apis/notify/const.go

@@ -62,6 +62,9 @@ const (
 	TEMPLATE_TYPE_CONTENT = "content"
 	TEMPLATE_TYPE_REMOTE  = "remote"
 
+	TEMPLATE_LANG_EN = "en"
+	TEMPLATE_LANG_CN = "cn"
+
 	CTYPE_ROBOT_YES  = "yes"
 	CTYPE_ROBOT_ONLY = "only"
 )

pkg/apis/scheduler/api.go

@@ -82,6 +82,8 @@ type ScheduleInput struct {
 	CpuMode      string `json:"cpu_mode"`
 	OsArch       string `json:"os_arch"`
 
+	SkipCpuCheck *bool `json:"skip_cpu_check"`
+
 	// In the migrate and create backup cases
 	// we don't need reallocate network
 	ReuseNetwork bool `json:"reuse_network"`

pkg/baremetal/profiles/profiles.go

@@ -27,7 +27,7 @@ type IPMIProfile struct {
 
 func DefaultProfile() IPMIProfile {
 	return IPMIProfile{
-		LanChannel: []int{1},
+		LanChannel: []int{1, 2, 8},
 		RootName:   "root",
 		RootId:     2,
 	}
@@ -84,14 +84,24 @@ func QemuProfile() IPMIProfile {
 	}
 }
 
+func H3CProfile() IPMIProfile {
+	return IPMIProfile{
+		LanChannel: []int{8, 1},
+		RootName:   "root",
+		RootId:     2,
+		StrongPass: true,
+	}
+}
+
 var (
 	PROFILES map[string]IPMIProfile = map[string]IPMIProfile{
-		"inspur":  InspurProfile(),
-		"lenovo":  LenovoProfile(),
-		"hp":      HpProfile(),
-		"huawei":  HuaweiProfile(),
-		"foxconn": FoxconnProfile(),
-		"qemu":    QemuProfile(),
+		types.OEM_NAME_INSPUR:  InspurProfile(),
+		types.OEM_NAME_LENOVO:  LenovoProfile(),
+		types.OEM_NAME_HP:      HpProfile(),
+		types.OEM_NAME_HUAWEI:  HuaweiProfile(),
+		types.OEM_NAME_FOXCONN: FoxconnProfile(),
+		types.OEM_NAME_QEMU:    QemuProfile(),
+		types.OEM_NAME_H3C:     H3CProfile(),
 	}
 )
 

pkg/baremetal/tasks/stop.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
@@ -47,7 +48,7 @@ func NewBaremetalServerStopTask(
 func (task *SBaremetalServerStopTask) DoStop(ctx context.Context, args interface{}) error {
 	task.SetStage(task.WaitForStop)
 	if err := task.Baremetal.DoPowerShutdown(true); err != nil {
-		return errors.Wrap(err, "Do power shutdown error")
+		log.Errorf("Do power shutdown error: %s", err)
 	}
 	task.startTime = time.Now()
 	ExecuteTask(task, nil)
@@ -73,7 +74,7 @@ func (self *SBaremetalServerStopTask) WaitForStop(ctx context.Context, args inte
 		isSoft = false
 	}
 	if err := self.Baremetal.DoPowerShutdown(isSoft); err != nil {
-		return errors.Wrap(err, "DoPowerShutdown")
+		log.Errorf("DoPowerShutdown soft=%v error: %s", isSoft, err)
 	}
 	time.Sleep(10 * time.Second)
 	ExecuteTask(self, nil)

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -115,6 +115,8 @@ func (dev *MegaRaidPhyDev) parseLine(line string) bool {
 			dev.Status = "jbod"
 		} else if strings.Contains(strings.ToLower(val), "online") {
 			dev.Status = "online"
+		} else if val == "Rebuild" {
+			dev.Status = "rebuild"
 		} else {
 			dev.Status = "offline"
 		}
@@ -624,7 +626,7 @@ func (a StorcliAdaptor) key() string {
 }
 
 func (a *StorcliAdaptor) isComplete() bool {
-	return a.Controller >= 0 && a.key() != ""
+	return a.Controller >= 0 && a.name != "" && a.sn != ""
 }
 
 func (a *StorcliAdaptor) parseLine(l string) {

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -56,7 +56,7 @@ func (dev *Mpt2SASRaidPhyDev) parseLine(line string) bool {
 	}
 	switch key {
 	case "Drive Type":
-		if strings.HasSuffix("_HDD", val) {
+		if strings.HasSuffix(val, "_HDD") {
 			dev.Rotate = tristate.True
 		} else {
 			dev.Rotate = tristate.False

pkg/cloudcommon/db/auto_delete.go

@@ -0,0 +1,54 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by model-api-gen. DO NOT EDIT.
+
+package db
+
+import (
+	"context"
+
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+// +onecloud:model-api-gen
+type SAutoDeleteResourceBase struct {
+	// 是否跟随资源自动删除
+	// example: false
+	AutoDelete bool `nullable:"false" default:"false" get:"user" update:"user" json:"auto_delete"`
+}
+
+type SAutoDeleteResourceBaseManager struct{}
+
+func (manager *SAutoDeleteResourceBaseManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query apis.AutoDeleteResourceBaseListInput,
+) (*sqlchemy.SQuery, error) {
+	if query.AutoDelete != nil {
+		q = q.Equals("auto_delete", *query.AutoDelete)
+	}
+
+	return q, nil
+}
+
+func (self *SAutoDeleteResourceBase) SetAutoDelete(model IModel, userCred mcclient.TokenCredential, autoDelete bool) error {
+	_, err := Update(model, func() error {
+		self.AutoDelete = autoDelete
+		return nil
+	})
+	return err
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -425,9 +425,6 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 					extraResults[i].Update(jsonDict)
 					jsonDict = extraResults[i]
 				}
-				if len(exportKeys) > 0 {
-					jsonDict = jsonDict.CopyIncludes(exportKeys...)
-				}
 				if len(fieldFilter) > 0 {
 					jsonDict = jsonDict.CopyIncludes(fieldFilter...)
 				}
@@ -791,6 +788,9 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if len(retList) != retCount {
 		totalCnt = len(retList)
 	}
+	if query.Contains("export_keys") {
+		retList = getExportCols(query, retList)
+	}
 	paginate := false
 	if !customizeFilters.IsEmpty() {
 		// query not use Limit and Offset, do manual pagination
@@ -822,6 +822,18 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 	return &retResult
 }
 
+func getExportCols(query jsonutils.JSONObject, retList []jsonutils.JSONObject) []jsonutils.JSONObject {
+	var exportKeys stringutils2.SSortedStrings
+	exportKeyStr, _ := query.GetString("export_keys")
+	exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ","))
+	for i := range retList {
+		if len(exportKeys) > 0 {
+			retList[i] = retList[i].(*jsonutils.JSONDict).CopyIncludes(exportKeys...)
+		}
+	}
+	return retList
+}
+
 func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modulebase.ListResult, error) {
 	userCred := fetchUserCredential(ctx)
 

pkg/cloudcommon/db/metadata.go

@@ -459,17 +459,17 @@ func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key s
 }
 
 func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
 		return err
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
 
-func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
+func (manager *SMetadataManager) setValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) {
 	idStr := GetObjectIdstr(obj)
 
 	// no need to lock
@@ -546,9 +546,9 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 }
 
 func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error {
-	changes, err := manager.SetValues(ctx, obj, store, userCred)
+	changes, err := manager.setValues(ctx, obj, store, userCred)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "setValues")
 	}
 
 	idStr := GetObjectIdstr(obj)
@@ -565,9 +565,9 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 	q := manager.Query().Equals("id", idStr).NotLike("key", `\_\_%`) //避免删除系统内置的metadata, _ 在mysql里面有特殊含义,需要转义
 	switch delRange {
 	case USER_TAG_PREFIX:
-		q = q.Like("key", USER_TAG_PREFIX+"%")
+		q = q.Startswith("key", USER_TAG_PREFIX)
 	case CLOUD_TAG_PREFIX:
-		q = q.Like("key", CLOUD_TAG_PREFIX+"%")
+		q = q.Startswith("key", CLOUD_TAG_PREFIX)
 	}
 	q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys)))
 	if err := FetchModelObjects(manager, q, &records); err != nil {
@@ -581,7 +581,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m
 		changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value})
 	}
 	if len(changes) > 0 {
-		OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
+		OpsLog.LogEvent(obj.GetIModel(), ACT_SET_METADATA, jsonutils.Marshal(changes), userCred)
 	}
 	return nil
 }
@@ -603,7 +603,7 @@ func (manager *SMetadataManager) GetAll(obj IModel, keys []string, keyPrefix str
 	ret := make(map[string]string)
 	for _, rec := range records {
 		if len(rec.Value) > 0 || strings.HasPrefix(rec.Key, USER_TAG_PREFIX) {
-			ret[rec.Key] = rec.Value
+			ret[strings.ToLower(rec.Key)] = rec.Value
 		}
 	}
 	return ret, nil

pkg/cloudcommon/db/opslog.go

@@ -126,7 +126,8 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
 	if !consts.OpsLogEnabled() {
 		return
 	}
-	if len(model.GetId()) == 0 || len(model.GetName()) == 0 {
+	if len(model.GetId()) == 0 {
+		log.Errorf("logevent for an object without ID???")
 		return
 	}
 	if action == ACT_UPDATE {

pkg/cloudcommon/db/opslog_const.go

@@ -47,8 +47,9 @@ const (
 	ACT_BACKUP_START        = "backup_start"
 	ACT_BACKUP_START_FAILED = "backup_start_fail"
 
-	ACT_FREEZE   = "freeze"
-	ACT_UNFREEZE = "unfreeze"
+	ACT_FREEZE      = "freeze"
+	ACT_FREEZE_FAIL = "freeze_fail"
+	ACT_UNFREEZE    = "unfreeze"
 
 	ACT_RESTARING    = "restarting"
 	ACT_RESTART_FAIL = "restart_fail"

pkg/cloudcommon/db/rbac.go

@@ -51,6 +51,8 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeUser:
 		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() || objOwnerId.GetUserId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && objOwnerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}
@@ -103,6 +105,8 @@ func isClassRbacAllowed(manager IModelManager, userCred mcclient.TokenCredential
 	case rbacutils.ScopeUser:
 		if ownerId != nil && ownerId.GetUserId() == objOwnerId.GetUserId() {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}

pkg/cloudcommon/etcd/etcd.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
@@ -45,7 +46,8 @@ type SEtcdClient struct {
 	onKeepaliveFailure func()
 	leaseLiving        bool
 
-	watchers map[string]*SEtcdWatcher
+	watchers   map[string]*SEtcdWatcher
+	watchersMu *sync.Mutex
 }
 
 func defaultOnKeepAliveFailed() {
@@ -115,6 +117,7 @@ func NewEtcdClient(opt *SEtcdOptions, onKeepaliveFailure func()) (*SEtcdClient,
 	etcdClient.leaseTtlTimeout = timeoutSeconds
 
 	etcdClient.watchers = make(map[string]*SEtcdWatcher)
+	etcdClient.watchersMu = &sync.Mutex{}
 
 	etcdClient.namespace = opt.EtcdNamspace
 
@@ -306,8 +309,10 @@ func (cli *SEtcdClient) Watch(
 	onModify TEtcdModifyEventFunc,
 	onDelete TEtcdDeleteEventFunc,
 ) error {
+	cli.watchersMu.Lock()
 	_, ok := cli.watchers[prefix]
 	if ok {
+		cli.watchersMu.Unlock()
 		return errors.Errorf("watch prefix %s already registered", prefix)
 	}
 
@@ -318,6 +323,7 @@ func (cli *SEtcdClient) Watch(
 		watcher: watcher,
 		cancel:  cancel,
 	}
+	cli.watchersMu.Unlock()
 
 	prefix = cli.getKey(prefix)
 
@@ -346,6 +352,8 @@ func (cli *SEtcdClient) Watch(
 }
 
 func (cli *SEtcdClient) Unwatch(prefix string) {
+	cli.watchersMu.Lock()
+	defer cli.watchersMu.Unlock()
 	watcher, ok := cli.watchers[prefix]
 	if ok {
 		log.Debugf("unwatch %s", prefix)

pkg/cloudcommon/notifyclient/events.go

@@ -47,6 +47,7 @@ var (
 	ActionPendingDelete SAction = "pending_delete"
 	ActionRebuildRoot   SAction = "rebuild_root"
 	ActionChangeConfig  SAction = "change_config"
+	ActionChangeIpaddr  SAction = "change_ipaddr"
 )
 
 type SEvent struct {

pkg/cloudcommon/notifyclient/notify.go

@@ -339,7 +339,11 @@ func genMsgViaLang(ctx context.Context, p sNotifyParams) ([]npk.SNotifyMessage,
 }
 
 func intelliNotify(ctx context.Context, p sNotifyParams) {
-	log.Infof("recipientId: %v, contacts: %v, event %s priority %s", p.recipientId, p.contacts, p.event, p.priority)
+	log.Infof("recipientId: %v, contactType: %s, contacts: %v, event %s priority %s", p.recipientId, p.channel, p.contacts, p.event, p.priority)
+	// hack
+	if len(p.recipientId) == 0 && len(p.contacts) == 0 {
+		p.contacts = []string{""}
+	}
 	msgs, err := genMsgViaLang(ctx, p)
 	if err != nil {
 		log.Errorf("unable send notification: %v", err)
@@ -433,14 +437,9 @@ func notifyWithChannel(ctx context.Context, p sNotifyParams, channels ...npk.TNo
 		p.recipientId = []string{}
 		p.contacts = []string{}
 		p.channel = c
-		if c == npk.NotifyByWebConsole {
-			p.contacts = reps
-		} else {
-			p.recipientId = reps
-		}
+		p.recipientId = reps
 		rawNotify(ctx, p)
 	}
-
 }
 
 func NotifyImportant(recipientId []string, isGroup bool, event string, data jsonutils.JSONObject) {
@@ -519,7 +518,7 @@ func notifyRobot(ctx context.Context, robot string, recipientId []string, isGrou
 	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
 	params := jsonutils.NewDict()
 	params.Set("robot", jsonutils.NewString(robot))
-	result, err := modules.NotifyConfig.PerformClassAction(s, "get-types", params)
+	result, err := modules.NotifyReceiver.PerformClassAction(s, "get-types", params)
 	if err != nil {
 		return err
 	}

pkg/cloudcommon/types/types.go

@@ -35,6 +35,7 @@ const (
 	OEM_NAME_FOXCONN    = "foxconn"
 	OEM_NAME_QEMU       = "qemu"
 	OEM_NAME_SUPERMICRO = "supermicro"
+	OEM_NAME_H3C        = "h3c"
 )
 
 var (
@@ -48,6 +49,7 @@ var (
 		OEM_NAME_FOXCONN,
 		OEM_NAME_QEMU,
 		OEM_NAME_SUPERMICRO,
+		OEM_NAME_H3C,
 	}
 )
 

pkg/cloudcommon/validators/validators.go

@@ -829,6 +829,9 @@ var ValidateModel = func(userCred mcclient.TokenCredential, manager db.IStandalo
 		if errors.Cause(err) == sql.ErrNoRows {
 			return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), *id)
 		}
+		if errors.Cause(err) == sqlchemy.ErrDuplicateEntry {
+			return nil, httperrors.NewDuplicateResourceError(manager.Keyword(), *id)
+		}
 		return nil, httperrors.NewGeneralError(err)
 	}
 	*id = model.GetId()

pkg/cloudevent/models/cloudevents.go

@@ -34,7 +34,7 @@ import (
 
 type SCloudeventManager struct {
 	db.SModelBaseManager
-	db.SProjectizedResourceBaseManager
+	db.SDomainizedResourceBaseManager
 }
 
 var CloudeventManager *SCloudeventManager
@@ -57,7 +57,7 @@ func init() {
 
 type SCloudevent struct {
 	db.SModelBase
-	db.SProjectizedResourceBase
+	db.SDomainizedResourceBase
 
 	EventId      int64                `primary:"true" auto_increment:"true" list:"user"`
 	Name         string               `width:"128" charset:"utf8" nullable:"false" index:"true" list:"user"`
@@ -76,6 +76,10 @@ type SCloudevent struct {
 	Brand           string `width:"64" charset:"ascii" list:"domain"`
 }
 
+func (self *SCloudeventManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsDomainAllowList(userCred, self)
+}
+
 func (self *SCloudeventManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return false
 }
@@ -88,6 +92,10 @@ func (self *SCloudevent) AllowUpdateItem(ctx context.Context, userCred mcclient.
 	return false
 }
 
+func (self *SCloudevent) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsDomainAllowGet(userCred, self)
+}
+
 // 云平台操作日志列表
 func (manager *SCloudeventManager) ListItemFilter(
 	ctx context.Context,
@@ -97,7 +105,11 @@ func (manager *SCloudeventManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	q, err := manager.SModelBaseManager.ListItemFilter(ctx, q, userCred, input.ModelBaseListInput)
 	if err != nil {
-		return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
+		return nil, errors.Wrap(err, "SModelBaseManager.ListItemFilter")
+	}
+	q, err = manager.SDomainizedResourceBaseManager.ListItemFilter(ctx, q, userCred, input.DomainizedResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SDomainizedResourceBaseManager.ListItemFilter")
 	}
 
 	if len(input.Providers) > 0 {
@@ -153,37 +165,45 @@ func (manager *SCloudeventManager) FetchCustomizeColumns(
 ) []api.CloudeventDetails {
 	rows := make([]api.CloudeventDetails, len(objs))
 	base := manager.SModelBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
-	projRows := manager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	domainRows := manager.SDomainizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range rows {
 		rows[i].ModelBaseDetails = base[i]
-		rows[i].ProjectizedResourceInfo = projRows[i]
+		rows[i].DomainizedResourceInfo = domainRows[i]
 	}
 	return rows
 }
 
+func (self *SCloudevent) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	return self.SModelBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
+}
+
+func (manager *SCloudeventManager) NamespaceScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeDomain
+}
+
 func (manager *SCloudeventManager) ResourceScope() rbacutils.TRbacScope {
-	return rbacutils.ScopeProject
+	return rbacutils.ScopeDomain
 }
 
 func (self *SCloudevent) GetOwnerId() mcclient.IIdentityProvider {
-	owner := db.SOwnerId{DomainId: self.DomainId, ProjectId: self.ProjectId}
+	owner := db.SOwnerId{DomainId: self.DomainId}
 	return &owner
 }
 
 func (manager *SCloudeventManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
-	return manager.SProjectizedResourceBaseManager.FilterByOwner(q, owner, scope)
+	return manager.SDomainizedResourceBaseManager.FilterByOwner(q, owner, scope)
 }
 
 func (manager *SCloudeventManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
-	return manager.SProjectizedResourceBaseManager.FetchOwnerId(ctx, data)
+	return manager.SDomainizedResourceBaseManager.FetchOwnerId(ctx, data)
 }
 
 func (manager *SCloudeventManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
-	return manager.SProjectizedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
+	return manager.SDomainizedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
 }
 
 func (manager *SCloudeventManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
-	return manager.SProjectizedResourceBaseManager.QueryDistinctExtraField(q, field)
+	return manager.SDomainizedResourceBaseManager.QueryDistinctExtraField(q, field)
 }
 
 func (manager *SCloudeventManager) OrderByExtraFields(
@@ -192,7 +212,7 @@ func (manager *SCloudeventManager) OrderByExtraFields(
 	userCred mcclient.TokenCredential,
 	query api.CloudeventListInput,
 ) (*sqlchemy.SQuery, error) {
-	return manager.SProjectizedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.ProjectizedResourceListInput)
+	return manager.SDomainizedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.DomainizedResourceListInput)
 }
 
 func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider *SCloudprovider, iEvents []cloudprovider.ICloudEvent) int {
@@ -213,7 +233,6 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred
 			CloudproviderId: cloudprovider.Id,
 		}
 		event.DomainId = cloudprovider.DomainId
-		event.ProjectId = cloudprovider.ProjectId
 		if len(event.Brand) == 0 {
 			event.Brand = event.Provider
 		}

pkg/cloudevent/models/cloudproviders.go

@@ -279,23 +279,12 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	if err != nil {
 		return start, end, errors.Wrap(err, "self.GetProviderFactory")
 	}
-	q := CloudeventManager.Query().Equals("cloudprovider_id", self.Id).Desc("created_at")
-	count, err := q.CountWithError()
-	if err != nil {
-		return start, end, errors.Wrap(err, "q.CountWithError")
-	}
 	if !self.LastSyncTimeAt.IsZero() {
 		start = self.LastSyncTimeAt
-	} else if count == 0 {
-		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	} else {
-		event := &SCloudevent{}
-		err = q.First(event)
-		if err != nil {
-			return start, end, errors.Wrap(err, "q.First")
-		}
-		start = event.CreatedAt
+		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	}
+
 	// 避免cloudevent过长时间未运行，再次运行时记录的最后一条时间距离现在间隔太长
 	if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
 		start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)

pkg/cloudevent/policy/resources.go

@@ -21,8 +21,10 @@ import (
 
 var (
 	cloudeventSystemResources = []string{}
-	cloudeventDomainResources = []string{}
-	cloudeventUserResources   = []string{}
+	cloudeventDomainResources = []string{
+		"cloudevents",
+	}
+	cloudeventUserResources = []string{}
 )
 
 func init() {

pkg/cloudid/models/cloudaccount.go

@@ -383,6 +383,10 @@ func (manager *SCloudaccountManager) SyncCloudaccounts(ctx context.Context, user
 }
 
 func (self *SCloudaccount) StartSyncSamlProvidersTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	if self.SAMLAuth.IsFalse() {
+		log.Debugf("cloudaccount %s(%s) not enable saml auth, skip sycing saml provider", self.Name, self.Provider)
+		return nil
+	}
 	params := jsonutils.NewDict()
 	task, err := taskman.TaskManager.NewTask(ctx, "SyncSAMLProvidersTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
@@ -1635,9 +1639,14 @@ func (self *SCloudaccount) SyncSystemCloudpoliciesForCloud(ctx context.Context,
 	return nil
 }
 
-func (self *SCloudaccount) GetLocalUserCloudroles(userId, spId string) ([]SCloudrole, error) {
+func (self *SCloudaccount) GetLocalCloudroles(userId, groupId string, spId string, grouped bool) ([]SCloudrole, error) {
 	roles := []SCloudrole{}
-	q := CloudroleManager.Query().Equals("cloudaccount_id", self.Id).Equals("owner_id", userId).Equals("saml_provider_id", spId)
+	q := CloudroleManager.Query().Equals("cloudaccount_id", self.Id).Equals("saml_provider_id", spId)
+	if grouped {
+		q = q.Equals("cloudgroup_id", groupId)
+	} else {
+		q = q.Equals("owner_id", userId)
+	}
 	err := db.FetchModelObjects(CloudroleManager, q, &roles)
 	if err != nil {
 		return nil, errors.Wrapf(err, "db.FetchModelObjects")
@@ -1645,55 +1654,90 @@ func (self *SCloudaccount) GetLocalUserCloudroles(userId, spId string) ([]SCloud
 	return roles, nil
 }
 
-func (self *SCloudaccount) RegisterCloudrole(userId, spId string) (*SCloudrole, error) {
-	roles, err := self.GetLocalUserCloudroles(userId, spId)
+func (self *SCloudaccount) RegisterCloudroles(userId string, grouped bool, spId string) ([]SCloudrole, error) {
+	samlUsers, err := self.GetSamlusers()
 	if err != nil {
-		return nil, errors.Wrapf(err, "GetLocalUserCloudroles")
+		return nil, errors.Wrapf(err, "GetSamlusers")
 	}
-	if len(roles) > 0 {
-		return &roles[0], nil
+	ret := []SCloudrole{}
+	roleIds := []string{}
+	for i := range samlUsers {
+		if samlUsers[i].OwnerId == userId {
+			roles, err := self.GetLocalCloudroles(userId, samlUsers[i].CloudgroupId, spId, grouped)
+			if err != nil {
+				return nil, errors.Wrapf(err, "GetLocalUserCloudroles")
+			}
+			for i := range roles {
+				if !utils.IsInStringArray(roles[i].Id, roleIds) {
+					ret = append(ret, roles[i])
+					break
+				}
+			}
+			if len(roles) == 0 {
+				role := SCloudrole{}
+				role.SetModelManager(CloudroleManager, &role)
+				role.CloudaccountId = self.Id
+				role.SAMLProviderId = spId
+				if grouped {
+					group, err := CloudgroupManager.FetchById(samlUsers[i].CloudgroupId)
+					if err != nil {
+						return nil, errors.Wrapf(err, "CloudgroupManager.FetchById(%s)", samlUsers[i].CloudgroupId)
+					}
+					role.Name = stringutils2.GenerateRoleName(group.GetName())
+					role.CloudgroupId = group.GetId()
+				} else {
+					user, err := db.UserCacheManager.FetchById(userId)
+					if err != nil {
+						return nil, errors.Wrapf(err, "UserCacheManager.FetchById(%s)", userId)
+					}
+					role.Name = stringutils2.GenerateRoleName(user.GetName())
+					role.OwnerId = userId
+				}
+				role.Status = api.CLOUD_ROLE_STATUS_CREATING
+				role.DomainId = self.DomainId
+				err = CloudroleManager.TableSpec().Insert(context.TODO(), &role)
+				if err != nil {
+					return nil, errors.Wrapf(err, "Insert role")
+				}
+				ret = append(ret, role)
+			}
+		}
 	}
-	user, err := db.UserCacheManager.FetchById(userId)
-	if err != nil {
-		return nil, errors.Wrapf(err, "UserCacheManager.FetchById(%s)", userId)
-	}
-	role := &SCloudrole{}
-	role.SetModelManager(CloudroleManager, role)
-	role.CloudaccountId = self.Id
-	role.OwnerId = userId
-	role.SAMLProviderId = spId
-	role.Name = stringutils2.GenerateRoleName(user.GetName())
-	role.Status = api.CLOUD_ROLE_STATUS_CREATING
-	role.DomainId = self.DomainId
-	return role, CloudroleManager.TableSpec().Insert(context.TODO(), role)
+	return ret, nil
 }
 
-func (self *SCloudaccount) GetCloudrole(userId string) (*SCloudrole, error) {
+func (self *SCloudaccount) getCloudrolesForSync(userId string, grouped bool) ([]SCloudrole, error) {
 	sp, valid := self.IsSAMLProviderValid()
 	if !valid {
 		return nil, fmt.Errorf("SAMLProvider for account %s not ready", self.Id)
 	}
 
-	return self.RegisterCloudrole(userId, sp.Id)
+	return self.RegisterCloudroles(userId, grouped, sp.Id)
 }
 
-func (self *SCloudaccount) SyncRole(userId string) (*SCloudrole, error) {
-	role, err := self.GetCloudrole(userId)
+func (self *SCloudaccount) SyncRoles(userId string, grouped bool) ([]SCloudrole, error) {
+	roles, err := self.getCloudrolesForSync(userId, grouped)
 	if err != nil {
 		return nil, errors.Wrapf(err, "GetCloudrole")
 	}
 
-	err = role.SyncRoles()
-	if err != nil {
-		return nil, errors.Wrapf(err, "SyncRoles")
+	for i := range roles {
+		err = roles[i].SyncRoles()
+		if err != nil {
+			return nil, errors.Wrapf(err, "SyncRoles")
+		}
 	}
 
-	return role, nil
+	if len(roles) == 0 {
+		return nil, fmt.Errorf("not found any available roles")
+	}
+
+	return roles, nil
 }
 
 func (self *SCloudaccount) GetCloudroles() ([]SCloudrole, error) {
 	roles := []SCloudrole{}
-	q := CloudroleManager.Query()
+	q := CloudroleManager.Query().Equals("cloudaccount_id", self.Id)
 	err := db.FetchModelObjects(CloudroleManager, q, &roles)
 	if err != nil {
 		return nil, errors.Wrapf(err, "db.FetchModelObjects")

pkg/cloudid/models/cloudrole.go

@@ -41,6 +41,7 @@ type SCloudroleManager struct {
 	db.SExternalizedResourceBaseManager
 	SCloudaccountResourceBaseManager
 	SAMLProviderResourceBaseManager
+	SCloudgroupResourceBaseManager
 }
 
 var CloudroleManager *SCloudroleManager
@@ -62,6 +63,7 @@ type SCloudrole struct {
 	db.SExternalizedResourceBase
 	SCloudaccountResourceBase
 	SAMLProviderResourceBase
+	SCloudgroupResourceBase
 
 	Document *jsonutils.JSONDict `length:"long" charset:"ascii" list:"domain" update:"domain" create:"domain_required"`
 	OwnerId  string              `width:"128" charset:"ascii" index:"true" list:"user" nullable:"false" create:"optional"`
@@ -80,6 +82,11 @@ func (manager *SCloudroleManager) ListItemFilter(ctx context.Context, q *sqlchem
 		return nil, err
 	}
 
+	q, err = manager.SCloudgroupResourceBaseManager.ListItemFilter(ctx, q, userCred, query.CloudgroupResourceListInput)
+	if err != nil {
+		return nil, err
+	}
+
 	return q, nil
 }
 
@@ -161,10 +168,13 @@ func (self *SCloudrole) GetICloudrole() (cloudprovider.ICloudrole, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "GetSAMLProvider")
 	}
-	for i := 0; i < 10; i++ {
+	for {
 		_, err := provider.GetICloudroleByName(self.Name)
-		if err != nil && errors.Cause(err) == cloudprovider.ErrNotFound {
-			break
+		if err != nil {
+			if errors.Cause(err) == cloudprovider.ErrNotFound {
+				break
+			}
+			return nil, errors.Wrapf(err, "GetICloudroleByName(%s)", self.Name)
 		}
 		info := strings.Split(self.Name, "-")
 		num, err := strconv.Atoi(info[len(info)-1])
@@ -196,10 +206,16 @@ func (self *SCloudrole) GetICloudrole() (cloudprovider.ICloudrole, error) {
 
 func (self *SCloudrole) GetCloudpolicies() ([]SCloudpolicy, error) {
 	q := CloudpolicyManager.Query()
-	samlUsers := SamluserManager.Query("cloudgroup_id").Equals("owner_id", self.OwnerId).Equals("cloudaccount_id", self.CloudaccountId).SubQuery()
-	groups := CloudgroupManager.Query("id").In("id", samlUsers)
-	gp := CloudgroupPolicyManager.Query("cloudpolicy_id").In("cloudgroup_id", groups).SubQuery()
-	q = q.In("id", gp)
+	var sq *sqlchemy.SSubQuery
+	if len(self.OwnerId) > 0 {
+		su := SamluserManager.Query("cloudgroup_id").Equals("owner_id", self.OwnerId).Equals("cloudaccount_id", self.CloudaccountId).SubQuery()
+		sq = CloudgroupPolicyManager.Query("cloudpolicy_id").In("cloudgroup_id", su).SubQuery()
+	} else if len(self.CloudgroupId) > 0 {
+		sq = CloudgroupPolicyManager.Query("cloudpolicy_id").Equals("cloudgroup_id", self.CloudgroupId).SubQuery()
+	} else {
+		return nil, fmt.Errorf("empty owner id or cloudgroup id")
+	}
+	q = q.In("id", sq)
 	policies := []SCloudpolicy{}
 	err := db.FetchModelObjects(CloudpolicyManager, q, &policies)
 	if err != nil {

pkg/cloudid/models/samluser.go

@@ -127,16 +127,6 @@ func (manager *SSamluserManager) ValidateCreateData(ctx context.Context, userCre
 		return input, err
 	}
 	group := _group.(*SCloudgroup)
-	sq := CloudgroupManager.Query("id").Equals("provider", group.Provider).SubQuery()
-	q := manager.Query().Equals("owner_id", input.OwnerId).In("cloudgroup_id", sq)
-	groups := []SCloudgroup{}
-	err = db.FetchModelObjects(CloudgroupManager, q, &groups)
-	if err != nil {
-		return input, httperrors.NewGeneralError(errors.Wrapf(err, "db.FetchModelObjects"))
-	}
-	if len(groups) > 0 {
-		return input, httperrors.NewConflictError("user %s has already in other %s group", input.Name, group.Provider)
-	}
 	_account, err := validators.ValidateModel(userCred, CloudaccountManager, &input.CloudaccountId)
 	if err != nil {
 		return input, err
@@ -148,6 +138,17 @@ func (manager *SSamluserManager) ValidateCreateData(ctx context.Context, userCre
 	if account.Provider != group.Provider {
 		return input, httperrors.NewConflictError("account %s and group %s not with same provider", account.Name, group.Name)
 	}
+
+	sq := CloudgroupManager.Query("id").Equals("provider", group.Provider).SubQuery()
+	q := manager.Query().Equals("owner_id", input.OwnerId).Equals("cloudaccount_id", account.Id).In("cloudgroup_id", sq)
+	groups := []SCloudgroup{}
+	err = db.FetchModelObjects(CloudgroupManager, q, &groups)
+	if err != nil {
+		return input, httperrors.NewGeneralError(errors.Wrapf(err, "db.FetchModelObjects"))
+	}
+	if len(groups) > 0 {
+		return input, httperrors.NewConflictError("user %s has already in other %s group", input.Name, group.Provider)
+	}
 	input.Status = api.SAML_USER_STATUS_AVAILABLE
 	return input, nil
 }

pkg/cloudid/saml/providers/aliyun/driver.go

@@ -52,7 +52,7 @@ func (d *SAliyunSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 		return data, httperrors.NewResourceNotReadyError("SAMLProvider for account %s not ready", account.Id)
 	}
 
-	role, err := account.SyncRole(userCred.GetUserId())
+	roles, err := account.SyncRoles(userCred.GetUserId(), true)
 	if err != nil {
 		return data, httperrors.NewGeneralError(errors.Wrapf(err, "SyncRole"))
 	}
@@ -61,7 +61,7 @@ func (d *SAliyunSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 	data.NameIdFormat = samlutils.NAME_ID_FORMAT_PERSISTENT
 	data.AudienceRestriction = sp.GetEntityId()
 	for k, v := range map[string]string{
-		"https://www.aliyun.com/SAML-Role/Attributes/Role":            fmt.Sprintf("%s,%s", role.ExternalId, SAMLProvider.ExternalId),
+		"https://www.aliyun.com/SAML-Role/Attributes/Role":            fmt.Sprintf("%s,%s", roles[0].ExternalId, SAMLProvider.ExternalId),
 		"https://www.aliyun.com/SAML-Role/Attributes/RoleSessionName": userCred.GetUserId(),
 		"https://www.aliyun.com/SAML-Role/Attributes/SessionDuration": "1800",
 	} {

pkg/cloudid/saml/providers/aws/driver.go

@@ -52,7 +52,7 @@ func (d *SAWSSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCred
 		return data, httperrors.NewResourceNotReadyError("SAMLProvider for account %s not ready", account.Id)
 	}
 
-	role, err := account.SyncRole(userCred.GetUserId())
+	roles, err := account.SyncRoles(userCred.GetUserId(), true)
 	if err != nil {
 		return data, httperrors.NewGeneralError(errors.Wrapf(err, "SyncRole"))
 	}
@@ -68,7 +68,7 @@ func (d *SAWSSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCred
 		{
 			name:         "https://aws.amazon.com/SAML/Attributes/Role",
 			friendlyName: "RoleEntitlement",
-			value:        fmt.Sprintf("%s,%s", role.ExternalId, SAMLProvider.ExternalId),
+			value:        fmt.Sprintf("%s,%s", roles[0].ExternalId, SAMLProvider.ExternalId),
 		},
 		{
 			name:         "https://aws.amazon.com/SAML/Attributes/RoleSessionName",

pkg/cloudid/saml/providers/awscn/driver.go

@@ -51,7 +51,7 @@ func (d *SAWSCNSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCre
 		return data, httperrors.NewResourceNotReadyError("SAMLProvider for account %s not ready", account.Id)
 	}
 
-	role, err := account.SyncRole(userCred.GetUserId())
+	roles, err := account.SyncRoles(userCred.GetUserId(), true)
 	if err != nil {
 		return data, httperrors.NewGeneralError(errors.Wrapf(err, "SyncRole"))
 	}
@@ -67,7 +67,7 @@ func (d *SAWSCNSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCre
 		{
 			name:         "https://aws.amazon.com/SAML/Attributes/Role",
 			friendlyName: "RoleEntitlement",
-			value:        fmt.Sprintf("%s,%s", role.ExternalId, SAMLProvider.ExternalId),
+			value:        fmt.Sprintf("%s,%s", roles[0].ExternalId, SAMLProvider.ExternalId),
 		},
 		{
 			name:         "https://aws.amazon.com/SAML/Attributes/RoleSessionName",

pkg/cloudid/saml/providers/qcloud/driver.go

@@ -52,14 +52,14 @@ func (d *SQcloudSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 		return data, httperrors.NewResourceNotReadyError("SAMLProvider for account %s not ready", account.Id)
 	}
 
-	role, err := account.SyncRole(userCred.GetUserId())
+	roles, err := account.SyncRoles(userCred.GetUserId(), true)
 	if err != nil {
 		return data, httperrors.NewGeneralError(errors.Wrapf(err, "SyncRole"))
 	}
 
-	roleStr := fmt.Sprintf("qcs::cam::uin/%s:roleName/%s,qcs::cam::uin/%s:saml-provider/%s", account.AccountId, role.ExternalId, account.AccountId, SAMLProvider.ExternalId)
+	roleStr := fmt.Sprintf("qcs::cam::uin/%s:roleName/%s,qcs::cam::uin/%s:saml-provider/%s", account.AccountId, roles[0].ExternalId, account.AccountId, SAMLProvider.ExternalId)
 
-	data.NameId = role.Name
+	data.NameId = roles[0].Name
 	data.NameIdFormat = samlutils.NAME_ID_FORMAT_TRANSIENT
 	data.AudienceRestriction = "https://cloud.tencent.com"
 	for _, v := range []struct {
@@ -75,7 +75,7 @@ func (d *SQcloudSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 		{
 			name:         "https://cloud.tencent.com/SAML/Attributes/RoleSessionName",
 			friendlyName: "RoleSessionName",
-			value:        role.Name,
+			value:        roles[0].Name,
 		},
 	} {
 		data.Attributes = append(data.Attributes, samlutils.SSAMLResponseAttribute{
@@ -94,7 +94,7 @@ func (d *SQcloudSAMLDriver) GetSpInitiatedLoginData(ctx context.Context, userCre
 	_account, err := models.CloudaccountManager.FetchById(cloudAccountId)
 	if err != nil {
 		if errors.Cause(err) == sql.ErrNoRows {
-			return data, httperrors.NewResourceNotFoundError("cloudaccount", cloudAccountId)
+			return data, httperrors.NewResourceNotFoundError2("cloudaccount", cloudAccountId)
 		}
 		return data, httperrors.NewGeneralError(err)
 	}
@@ -111,14 +111,14 @@ func (d *SQcloudSAMLDriver) GetSpInitiatedLoginData(ctx context.Context, userCre
 		return data, httperrors.NewResourceNotReadyError("SAMLProvider for account %s not ready", account.Id)
 	}
 
-	role, err := account.SyncRole(userCred.GetUserId())
+	roles, err := account.SyncRoles(userCred.GetUserId(), true)
 	if err != nil {
 		return data, httperrors.NewGeneralError(errors.Wrapf(err, "SyncRole"))
 	}
 
-	roleStr := fmt.Sprintf("qcs::cam::uin/%s:roleName/%s,qcs::cam::uin/%s:saml-provider/%s", account.AccountId, role.ExternalId, account.AccountId, SAMLProvider.ExternalId)
+	roleStr := fmt.Sprintf("qcs::cam::uin/%s:roleName/%s,qcs::cam::uin/%s:saml-provider/%s", account.AccountId, roles[0].ExternalId, account.AccountId, SAMLProvider.ExternalId)
 
-	data.NameId = role.Name
+	data.NameId = roles[0].Name
 	data.NameIdFormat = samlutils.NAME_ID_FORMAT_TRANSIENT
 	data.AudienceRestriction = "https://cloud.tencent.com"
 	for _, v := range []struct {
@@ -134,7 +134,7 @@ func (d *SQcloudSAMLDriver) GetSpInitiatedLoginData(ctx context.Context, userCre
 		{
 			name:         "https://cloud.tencent.com/SAML/Attributes/RoleSessionName",
 			friendlyName: "RoleSessionName",
-			value:        role.Name,
+			value:        roles[0].Name,
 		},
 	} {
 		data.Attributes = append(data.Attributes, samlutils.SSAMLResponseAttribute{

pkg/cloudprovider/cloudprovider.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -163,12 +162,6 @@ type ProviderConfig struct {
 	ProxyFunc httputils.TransportProxyFunc
 }
 
-func (cp *ProviderConfig) HttpClient() *http.Client {
-	client := httputils.GetClient(true, 15*time.Second)
-	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
-	return client
-}
-
 func (cp *ProviderConfig) AdaptiveTimeoutHttpClient() *http.Client {
 	client := httputils.GetAdaptiveTimeoutClient()
 	httputils.SetClientProxyFunc(client, cp.ProxyFunc)

pkg/cloudprovider/dbinstance.go

@@ -77,10 +77,7 @@ type SManagedDBInstanceCreateConfig struct {
 
 type SManagedDBInstanceChangeConfig struct {
 	DiskSizeGB   int
-	StorageType  string
 	InstanceType string
-	VcpuCount    int
-	VmemSizeMb   int
 }
 
 type SDBInstanceDatabaseCreateConfig struct {

pkg/cloudprovider/objectstore.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"reflect"
 	"regexp"
 	"sort"
 	"strconv"
@@ -110,12 +111,15 @@ type SBucketCORSRule struct {
 }
 
 type SBucketRefererConf struct {
-	// 白名单域名列表
-	WhiteList []string
-	// 黑名单域名列表
-	BlackList []string
+	// 域名列表
+	DomainList []string
+	// 域名列表
+	// enmu: Black-List, White-List
+	RefererType string
 	// 是否允许空referer 访问
 	AllowEmptyRefer bool
+
+	Enabled bool
 }
 
 type SBucketPolicyStatement struct {
@@ -264,8 +268,7 @@ type ICloudBucket interface {
 	DeletePolicy(id []string) ([]SBucketPolicyStatement, error)
 
 	GetTags() (map[string]string, error)
-	SetTags(tags map[string]string) error
-	DeleteTags() error
+	SetTags(tags map[string]string, replace bool) error
 }
 
 type ICloudObject interface {
@@ -829,32 +832,27 @@ func DeleteBucketCORS(ibucket ICloudBucket, id []string) ([]SBucketCORSRule, err
 	return deletedRules, nil
 }
 
-func SetBucketMetadata(ibucket ICloudBucket, tags map[string]string, replace bool) error {
-	newTags := map[string]string{}
-	if replace {
-		newTags = tags
-	} else {
-		oldTags, err := ibucket.GetTags()
-		if err != nil {
-			return errors.Wrap(err, "b.getTags()")
-		}
-		for k, v := range oldTags {
-			if _, ok := tags[k]; !ok {
-				tags[k] = v
-			}
-		}
-		newTags = tags
-	}
-	if len(newTags) == 0 {
-		err := ibucket.DeleteTags()
-		if err != nil {
-			return errors.Wrap(err, "b.DeleteTags()")
-		}
-	} else {
-		err := ibucket.SetTags(newTags)
-		if err != nil {
-			return errors.Wrapf(err, "b.setTags(%s)", jsonutils.Marshal(newTags).String())
-		}
-	}
-	return nil
+type TagsUpdateInfo struct {
+	OldTags map[string]string
+	NewTags map[string]string
+}
+
+func (t TagsUpdateInfo) IsChanged() bool {
+	return !reflect.DeepEqual(t.OldTags, t.NewTags)
+}
+
+func SetBucketTags(iBucket ICloudBucket, tags map[string]string) (TagsUpdateInfo, error) {
+	ret := TagsUpdateInfo{}
+	old, err := iBucket.GetTags()
+	if err != nil {
+		if errors.Cause(err) == ErrNotImplemented || errors.Cause(err) == ErrNotSupported {
+			return ret, nil
+		}
+		return ret, errors.Wrapf(err, "iBucket.GetTags")
+	}
+	ret.OldTags, ret.NewTags = old, tags
+	if !ret.IsChanged() {
+		return ret, nil
+	}
+	return ret, iBucket.SetTags(tags, true)
 }