Merge pull request #5700 from swordqiu/automated-cherry-pick-of-#5698-upstream-release-3.0

Automated cherry pick of #5698: fix: service config changes not effective

.github/workflows/docker.yml

@@ -39,6 +39,7 @@ jobs:
         make cmd/baremetal-agent && ./scripts/bundle_libraries.sh _output/bin/bundles/baremetal-agent _output/bin/baremetal-agent
         make cmd/cloudevent cmd/devtool
         make cmd/*cli
+        make cmd/esxi-agent
 
     - name: Image baremetal-agent
       uses: elgohr/Publish-Docker-Github-Action@master
@@ -50,6 +51,16 @@ jobs:
         snapshot: true
         dockerfile: build/docker/Dockerfile.baremetal-agent
 
+    - name: Image esxi-agent
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: registry.cn-beijing.aliyuncs.com/yunionio/esxi-agent
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: registry.cn-beijing.aliyuncs.com
+        snapshot: true
+        dockerfile: build/docker/Dockerfile.esxi-agent
+
     - name: Image climc
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
@@ -180,7 +191,7 @@ jobs:
         snapshot: true
         dockerfile: build/docker/Dockerfile.host-deployer
 
-    - name: Image cloudevent 
+    - name: Image cloudevent
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
         name: registry.cn-beijing.aliyuncs.com/yunionio/cloudevent

Makefile

@@ -83,6 +83,9 @@ test:
 vet:
 	go vet ./...
 
+cmd/esxi-agent: prepare_dir
+	CGO_ENABLED=0 $(GO_BUILD) -o $(BIN_DIR)/$(shell basename $@) $(REPO_PREFIX)/$@
+
 cmd/%: prepare_dir
 	$(GO_BUILD) -o $(BIN_DIR)/$(shell basename $@) $(REPO_PREFIX)/$@
 

build/climc/root/opt/yunion/scripts/tools/clean_account.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+pushd $(dirname $BASH_SOURCE) > /dev/null
+CUR_DIR=$(pwd -P)
+popd > /dev/null
+
+
+ACCOUNT=$1
+
+if [ -z "$ACCOUNT" ]; then
+    echo "Usage: $0 <cloud-acount-id-or-name>"
+    exit 1
+fi
+
+ACCOUNT_ID=$(climc cloud-account-show $ACCOUNT | grep -w " id " | awk '{print $4}')
+ACCOUNT_NAME=$(climc cloud-account-show $ACCOUNT | grep -w " name " | awk '{print $4}')
+
+if [ -z "$ACCOUNT_ID" ]; then
+    echo "Cloud account $ACCOUNT not found"
+    exit 1
+fi
+
+
+echo "Ready to clean cloud account $ACCOUNT_NAME($ACCOUNT_ID)..."
+
+echo "1. Disable cloud account $ACCOUNT_NAME($ACCOUNT_ID) ..."
+
+climc cloud-account-disable $ACCOUNT_ID
+
+echo "2. Clean all providers ..."
+
+for provider_id in $(climc cloud-provider-list --account $ACCOUNT_ID --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    echo "### to clean cloud provider $provider_id ..."
+    $CUR_DIR/clean_provider.sh $provider_id
+done
+
+echo "3. Delete cloud account ..."
+
+climc cloud-account-delete $ACCOUNT_ID

build/climc/root/opt/yunion/scripts/tools/clean_eip.sh

@@ -0,0 +1,27 @@
+#/bin/bash
+
+set -e
+
+EIP=$1
+
+if [ -z "$EIP" ]; then
+    echo "Usage: $0 <eip_id_or_name>"
+    exit 1
+fi
+
+EIP_NAME=$(climc eip-show $EIP | grep -w " name " | awk '{print $4}')
+EIP_ID=$(climc eip-show $EIP | grep -w " id " | awk '{print $4}')
+
+if [ -z "$EIP_ID" ]; then
+    echo "Cannot find eip $EIP_ID"
+    exit 1
+fi
+
+echo "To clean eip $EIP_NAME($EIP_ID)..."
+
+clean_eip() {
+    local EID=$1
+    climc eip-purge $EID
+}
+
+clean_eip $EIP_ID

build/climc/root/opt/yunion/scripts/tools/clean_host.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+
+HOST=$1
+
+if [ -z "$HOST" ]; then
+    echo "Usage: $0 <host_id_or_name>"
+    exit 1
+fi
+
+error_exit() {
+    echo "Error: $1"
+    exit 1
+}
+
+HOSTID=$(climc host-show $HOST | grep -w " id " | awk '{print $4}')
+
+HOSTNAME=$(climc host-show $HOST | grep -w " name " | awk '{print $4}')
+
+if [ -z "$HOSTNAME" ]; then
+    error_exit "Cannot find host $HOST"
+fi
+
+if [ -z "$HOSTID" ]; then
+    error_exit "Cannot find host $HOST"
+fi
+
+echo "Ready to purge host $HOSTNAME($HOSTID)..."
+
+echo "1. Disable $HOSTNAME($HOSTID)"
+
+if ! climc host-disable $HOSTID > /dev/null; then
+    error_exit "Fail to disable $HOST"
+fi
+
+echo "2. Clean servers on $HOSTNAME($HOSTID)"
+
+for i in $(climc server-list --host $HOSTID --admin --limit 2048 --system | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    climc server-update --delete enable $i > /dev/null
+    climc server-purge $i > /dev/null
+done
+
+for i in $(climc server-list --host $HOSTID --admin --limit 2048 --pending-delete --system | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    climc server-purge $i > /dev/null
+done
+
+echo "3. Clean disks on $HOSTNAME($HOSTID)"
+
+for storageid in $(climc host-storage-list --host $HOSTID --show-emulated | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $4}')
+do
+    STORAGENAME=$(climc storage-show $storageid | grep -w " name " | awk '{print $4}')
+    STORAGE_TYPE=$(climc storage-show $storageid | grep -w " storage_type " | awk '{print $4}')
+    if [ "$STORAGE_TYPE" == "local" ]; then
+        for diskid in $(climc disk-list --admin --system --details --search $storageid --limit 2048 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+        do
+            climc disk-purge $diskid > /dev/null
+        done
+        for diskid in $(climc disk-list --admin --system --details --search $storageid --pending-delete --limit 2048 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+        do
+            climc disk-purge $diskid > /dev/null
+        done
+    fi
+done
+
+echo "4. Remove host $HOSTNAME($HOSTID)"
+
+BAREMETAL=$(climc host-show $HOSTID | grep -w " is_baremetal " | awk '{print $4}')
+
+HOSTTYPE=$(climc host-show $HOSTID | grep -w " host_type " | awk '{print $4}')
+
+if [ "$BAREMETAL" == "true" ] && [ "$HOSTTYPE" != "baremetal" ]; then
+    climc host-update --host-type baremetal $HOSTID > /dev/null
+fi
+
+climc host-delete $HOSTID > /dev/null
+
+echo "DONE."

build/climc/root/opt/yunion/scripts/tools/clean_loadbalancer.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+LB=$1
+
+if [ -z "$LB" ]; then
+    echo "Usage: $0 <loadbalancer_id_or_name>"
+    exit 1
+fi
+
+LB_ID=$(climc lb-show $LB | grep -w " id " | awk '{print $4}')
+LB_NAME=$(climc lb-show $LB | grep -w " name " | awk '{print $4}')
+
+if [ -z "$LB_ID" ]; then
+    echo "Loadbalancer $LB not found"
+    exit 1
+fi
+
+echo "Ready to remove loadbalancer $LB_NAME($LB_ID) ..."
+
+clean_loadbalancer() {
+    local LID=$1
+    climc lb-purge $LID
+}
+
+clean_loadbalancer $LB_ID

build/climc/root/opt/yunion/scripts/tools/clean_loadbalancer_acl.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+LB_ACL=$1
+
+if [ -z "$LB_ACL" ]; then
+    echo "Usage: $0 <loadbalancer_id_or_name>"
+    exit 1
+fi
+
+LB_ACL_ID=$(climc lbacl-show $LB_ACL | grep -w " id " | awk '{print $4}')
+LB_ACL_NAME=$(climc lbacl-show $LB_ACL | grep -w " name " | awk '{print $4}')
+
+if [ -z "$LB_ACL_ID" ]; then
+    echo "Loadbalancer ACL $LB_ACL_ID not found"
+    exit 1
+fi
+
+echo "Ready to remove loadbalancer acl $LB_ACL_NAME($LB_ACL_ID) ..."
+
+clean_loadbalancer_acl() {
+    local LID=$1
+    climc lbacl-purge $LID
+}
+
+clean_loadbalancer_acl $LB_ACL_ID

build/climc/root/opt/yunion/scripts/tools/clean_loadbalancer_certificate.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+LB_CERT=$1
+
+if [ -z "$LB_CERT" ]; then
+    echo "Usage: $0 <loadbalancer_id_or_name>"
+    exit 1
+fi
+
+LB_CERT_ID=$(climc lbcert-show $LB_CERT | grep -w " id " | awk '{print $4}')
+LB_CERT_NAME=$(climc lbcert-show $LB_CERT | grep -w " name " | awk '{print $4}')
+
+if [ -z "$LB_CERT_ID" ]; then
+    echo "Loadbalancer certificate $LB_CERT_ID not found"
+    exit 1
+fi
+
+echo "Ready to remove loadbalancer certificate $LB_CERT_NAME($LB_CERT_ID) ..."
+
+clean_loadbalancer_cert() {
+    local LID=$1
+    climc lbcert-purge $LID
+}
+
+clean_loadbalancer_cert $LB_CERT_ID

build/climc/root/opt/yunion/scripts/tools/clean_provider.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+set -e
+
+pushd $(dirname $BASH_SOURCE) > /dev/null
+CUR_DIR=$(pwd -P)
+popd > /dev/null
+
+
+PROVIDER=$1
+
+if [ -z "$PROVIDER" ]; then
+    echo "Usage: $0 <cloud-provider-id-or-name>"
+    exit 1
+fi
+
+PROVIDER_ID=$(climc cloud-provider-show $PROVIDER | grep -w " id " | awk '{print $4}')
+PROVIDER_NAME=$(climc cloud-provider-show $PROVIDER | grep -w " name " | awk '{print $4}')
+
+if [ -z "$PROVIDER_ID" ]; then
+    echo "Cloud provider $PROVIDER not found"
+    exit 1
+fi
+
+
+echo "Ready to clean cloud provider $PROVIDER_NAME($PROVIDER_ID)..."
+
+echo "1. Disable cloud provider $PROVIDER_NAME($PROVIDER_ID) ..."
+
+climc cloud-provider-disable $PROVIDER_ID
+
+echo "2. Clean all hosts ..."
+
+for hostid in $(climc host-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_host.sh $hostid
+done
+
+echo "3. Clean all loadbalancer ..."
+
+for lbid in $(climc lb-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_loadbalancer.sh $lbid
+done
+
+echo "4. Clean all loadbalancer acl ..."
+
+for lbaclid in $(climc lbacl-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_loadbalancer_acl.sh $lbaclid
+done
+
+echo "5. Clean all loadbalancer certificate ..."
+
+for lbcertid in $(climc lbcert-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_loadbalancer_certificate.sh $lbcertid
+done
+
+echo "6. Clean all vpc ..."
+
+for vpcid in $(climc vpc-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_vpc.sh $vpcid
+done
+
+echo "7. Clean all eips ..."
+for eipid in $(climc eip-list --admin --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_eip.sh $eipid
+done
+
+echo "8. Clean all snapshots ..."
+for snapid in $(climc snapshot-list --admin --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    echo "to clean snapshot $snapid ..."
+    $CUR_DIR/clean_snapshot.sh $snapid
+done
+
+echo "9. Clean all storagecache ..."
+
+for cacheid in $(climc storage-cache-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_storagecache.sh $cacheid
+done
+
+echo "10. Clean all storages ..."
+
+for storeid in $(climc storage-list --manager $PROVIDER_ID --admin --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    $CUR_DIR/clean_storage.sh $storeid
+done
+
+echo "Delete cloud provider ..."
+
+climc cloud-provider-delete $PROVIDER_ID

build/climc/root/opt/yunion/scripts/tools/clean_snapshot.sh

@@ -0,0 +1,27 @@
+#/bin/bash
+
+set -e
+
+SNAPSHOT=$1
+
+if [ -z "$SNAPSHOT" ]; then
+    echo "Usage: $0 <snapshot_id_or_name>"
+    exit 1
+fi
+
+SNAPSHOT_NAME=$(climc snapshot-show $SNAPSHOT | grep -w " name " | awk '{print $4}')
+SNAPSHOT_ID=$(climc snapshot-show $SNAPSHOT | grep -w " id " | awk '{print $4}')
+
+if [ -z "$SNAPSHOT_ID" ]; then
+    echo "Cannot find snapshot $SNAPSHOT_ID"
+    exit 1
+fi
+
+echo "To clean snapshot $SNAPSHOT_NAME($SNAPSHOT_ID)..."
+
+clean_snapshot() {
+    local SNAP_ID=$1
+    climc snapshot-purge $SNAP_ID > /dev/null
+}
+
+clean_snapshot $SNAPSHOT_ID

build/climc/root/opt/yunion/scripts/tools/clean_storage.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+STORAGE=$1
+
+if [ -z "$STORAGE" ]; then
+    echo "Usage: $0 <storage_id_or_name>"
+    exit 1
+fi
+
+STORAGE_ID=$(climc storage-show $STORAGE | grep -w " id " | awk '{print $4}')
+STORAGE_NAME=$(climc storage-show $STORAGE | grep -w " name " | awk '{print $4}')
+
+if [ -z "$STORAGE_ID" ]; then
+    echo "Storage $STORAGE not found"
+    exit 1
+fi
+
+echo "Clean disks on storage $STORAGE_NAME($STORAGE_ID) ..."
+
+for disk_id in $(climc disk-list --admin --storage $STORAGE_ID --show-emulated --limit 2048 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    climc disk-purge $disk_id > /dev/null
+done
+
+for disk_id in $(climc disk-list --admin --storage $STORAGE_ID --show-emulated --pending-delete --limit 2048 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    climc disk-purge $disk_id > /dev/null
+done
+
+# CACHE_ID=$(climc storage-show $STORAGE_ID | grep -w " storagecache_id " | awk '{print $4}')
+
+climc storage-delete $STORAGE_ID > /dev/null

build/climc/root/opt/yunion/scripts/tools/clean_storagecache.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+CACHE=$1
+
+if [ -z "$CACHE" ]; then
+    echo "Usage: $0 <storage_cache_id_or_name>"
+    exit 1
+fi
+
+CACHE_ID=$(climc storage-cache-show $CACHE | grep -w " id " | awk '{print $4}')
+CACHE_NAME=$(climc storage-cache-show $CACHE | grep -w " name " | awk '{print $4}')
+
+if [ -z "$CACHE_ID" ]; then
+    echo "Storage cache $CACHE not found"
+    exit 1
+fi
+
+echo "Ready to remove storagecache $CACHE_NAME($CACHE_ID) ..."
+
+for imgid in $(climc storage-cached-image-list --storagecache $CACHE_ID --show-emulated --limit 0 | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $4}')
+do
+    climc storagecache-uncache-image $CACHE_ID $imgid --force # > /dev/null
+done
+
+climc storage-cache-delete $CACHE_ID > /dev/null
+

build/climc/root/opt/yunion/scripts/tools/clean_vpc.sh

@@ -0,0 +1,55 @@
+#/bin/bash
+
+set -e
+
+VPC=$1
+
+if [ -z "$VPC" ]; then
+    echo "Usage: $0 <vpc_id_or_name>"
+    exit 1
+fi
+
+VPC_NAME=$(climc vpc-show $VPC | grep -w " name " | awk '{print $4}')
+VPC_ID=$(climc vpc-show $VPC | grep -w " id " | awk '{print $4}')
+
+if [ -z "$VPC_ID" ]; then
+    echo "Cannot find vpc $VPC"
+    exit 1
+fi
+
+echo "To clean vpc $VPC_NAME($VPC_ID)..."
+
+clean_network() {
+    climc network-purge $1 > /dev/null
+}
+
+
+clean_wire() {
+    local WIRE_ID=$1
+    for net_id in $(climc network-list --admin --wire $WIRE_ID --show-emulated | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+    do
+        clean_network $net_id
+    done
+    climc wire-delete $wire_id > /dev/null
+}
+
+clean_vpc() {
+    local VPC_ID=$1
+    climc vpc-purge $VPC_ID > /dev/null
+}
+
+echo "1. Clean wires ..."
+
+for wire_id in $(climc wire-list --vpc $VPC_ID --show-emulated | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    clean_wire $wire_id
+done
+
+echo "2. Clean route tables ..."
+for routetable_id in $(climc routetable-list --vpc $VPC_ID --show-emulated | egrep '[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}' | awk '{print $2}')
+do
+    climc routetable-purge $routetable_id >/dev/null
+done
+
+echo "3. clean vpc ..."
+clean_vpc $VPC_ID

build/docker/Dockerfile.ansibleserver

@@ -1,15 +1,3 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
-
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk update && \
-    apk add --no-cache tzdata ansible ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:latest
 
 ADD ./_output/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -0,0 +1,13 @@
+FROM frolvlad/alpine-glibc:glibc-2.28
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk update && \
+    apk add --no-cache tzdata ansible ca-certificates && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.baremetal-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:latest
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.0.1
 
 MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
 

build/docker/Dockerfile.baremetal-base

@@ -27,3 +27,8 @@ COPY ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal
+
+# add mkisofs with bundles libraries
+ADD ./mkisofs /bin/mkisofs
+ADD ./.mkisofs.bin /bin/.mkisofs.bin
+ADD ./bundles/mkisofs /bin/bundles/mkisofs

build/docker/Dockerfile.climc

@@ -1,16 +1,5 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
+FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:v20200308
 
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN apk add --no-cache bash bash-completion tzdata ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV PATH="/opt/yunion/bin:${PATH}"
-
-RUN mkdir -p /opt/yunion/bin
+ADD ./build/climc/root/opt /opt
 
 ADD ./_output/bin/climc ./_output/bin/*cli /opt/yunion/bin/

build/docker/Dockerfile.climc-base

@@ -0,0 +1,16 @@
+FROM frolvlad/alpine-glibc:glibc-2.28
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk add --no-cache bash bash-completion tzdata ca-certificates vim && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+ENV PATH="/opt/yunion/bin:${PATH}"
+
+RUN mkdir -p /opt/yunion/bin

build/docker/Dockerfile.esxi-agent

@@ -0,0 +1,4 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+
+COPY ./build/esxi-agent/root/opt/ /opt/
+ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.host

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/host-base
+FROM registry.cn-beijing.aliyuncs.com/yunionio/host-base:v0.0.1
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 

build/docker/Dockerfile.host-deployer

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base
+FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:v0.0.2
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 

build/docker/Dockerfile.webconsole

@@ -1,14 +1,3 @@
-FROM yunion/kubectl:1.14.3
-
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk add --no-cache sshpass ipmitool openssh-client tzdata ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+FROM registry.cn-beijing.aliyuncs.com/yunionio/webconsole-base:latest
 
 ADD ./_output/bin/webconsole /opt/yunion/bin/webconsole

build/docker/Dockerfile.webconsole-base

@@ -0,0 +1,12 @@
+FROM yunion/kubectl:1.14.3
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk add --no-cache sshpass ipmitool openssh-client tzdata ca-certificates && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/esxi-agent/root/opt/yunion/share/vmware/ovf.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Envelope xmlns="http://schemas.dmtf.org/ovf/envelope/1"
+          xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
+          xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common"
+          xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
+          xmlns:vmw="http://www.vmware.com/schema/ovf"
+          xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <References>
+    <File ovf:href="{{ .Name }}" ovf:id="file1" ovf:size="{{ .Size }}"/>
+  </References>
+  <DiskSection>
+    <Info>Virtual disk information</Info>
+    <Disk ovf:capacity="{{ .Capacity }}" ovf:capacityAllocationUnits="byte" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized" ovf:populatedSize="0"/>
+  </DiskSection>
+  <VirtualSystem ovf:id="{{ .ImportName }}">
+    <Info>A virtual machine</Info>
+    <Name>{{ .ImportName }}</Name>
+    <OperatingSystemSection ovf:id="100" vmw:osType="other26xLinux64Guest">
+      <Info>The kind of installed guest operating system</Info>
+    </OperatingSystemSection>
+    <VirtualHardwareSection>
+      <Info>Virtual hardware requirements</Info>
+      <System>
+        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
+        <vssd:InstanceID>0</vssd:InstanceID>
+        <vssd:VirtualSystemIdentifier>{{ .ImportName }}</vssd:VirtualSystemIdentifier>
+        <vssd:VirtualSystemType>vmx-07</vssd:VirtualSystemType>
+      </System>
+      <Item>
+        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
+        <rasd:Description>Number of Virtual CPUs</rasd:Description>
+        <rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
+        <rasd:InstanceID>1</rasd:InstanceID>
+        <rasd:ResourceType>3</rasd:ResourceType>
+        <rasd:VirtualQuantity>1</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
+        <rasd:Description>Memory Size</rasd:Description>
+        <rasd:ElementName>1024MB of memory</rasd:ElementName>
+        <rasd:InstanceID>2</rasd:InstanceID>
+        <rasd:ResourceType>4</rasd:ResourceType>
+        <rasd:VirtualQuantity>1024</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:Address>0</rasd:Address>
+        <rasd:Description>SCSI Controller</rasd:Description>
+        <rasd:ElementName>SCSI Controller 0</rasd:ElementName>
+        <rasd:InstanceID>3</rasd:InstanceID>
+        <rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
+        <rasd:ResourceType>6</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:ElementName>Hard Disk 1</rasd:ElementName>
+        <rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
+        <rasd:InstanceID>9</rasd:InstanceID>
+        <rasd:Parent>3</rasd:Parent>
+        <rasd:ResourceType>17</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false"/>
+      </Item>
+    </VirtualHardwareSection>
+  </VirtualSystem>
+</Envelope>

build/esxi-agent/vars

@@ -0,0 +1,2 @@
+DESCRIPTION="Esxi Agent"
+# SERVICE="yes"

cmd/climc/shell/bucket_options.go

@@ -0,0 +1,46 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type BucketOptionsVerifyOptions struct {
+		options.BaseListOptions
+		CloudaccountId       string `help:"cloudaccount Id" required:"true"`
+		BillingReportBucket  string `help:"S3 bucket that stores account billing report"`
+		BillingFilePrefix    string `help:"prefix of billing file name"`
+		BillingBucketAccount string `help:"Id of cloudaccount that can access billing bucket"`
+		UsageReportBucket    string `help:"S3 bucket that stores account usage report"`
+		UsageFilePrefix      string `help:"prefix of usage file name"`
+		UsageBucketAccount   string `help:"Id of cloudaccount that can access usage bucket"`
+	}
+	R(&BucketOptionsVerifyOptions{}, "bucket-options-verify", "billing bucket connection test",
+		func(s *mcclient.ClientSession, args *BucketOptionsVerifyOptions) error {
+			params := jsonutils.Marshal(args)
+			result, err := modules.BucketOptions.PerformAction(s, args.CloudaccountId, "verify", params)
+			if err != nil {
+				return err
+			}
+			printObject(result)
+			return nil
+		})
+}

cmd/climc/shell/cloudaccounts.go

@@ -88,6 +88,19 @@ func init() {
 	R(&options.SAliyunCloudAccountCreateOptions{}, "cloud-account-create-aliyun", "Create an Aliyun cloud account", func(s *mcclient.ClientSession, args *options.SAliyunCloudAccountCreateOptions) error {
 		params := jsonutils.Marshal(args)
 		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Aliyun"), "provider")
+		options := jsonutils.NewDict()
+		if len(args.OptionsBillingReportBucket) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
+		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFilePrefix) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFilePrefix), "billing_file_prefix")
+		}
+		if options.Size() > 0 {
+			params.(*jsonutils.JSONDict).Add(options, "options")
+		}
 		result, err := modules.Cloudaccounts.Create(s, params)
 		if err != nil {
 			return err
@@ -147,6 +160,12 @@ func init() {
 		if len(args.OptionsBillingReportBucket) > 0 {
 			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
 		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFilePrefix) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFilePrefix), "billing_file_prefix")
+		}
 		if options.Size() > 0 {
 			params.Add(options, "options")
 		}
@@ -280,6 +299,34 @@ func init() {
 		if params.Size() == 0 {
 			return InvalidUpdateError()
 		}
+
+		options := jsonutils.NewDict()
+		if len(args.OptionsBillingReportBucket) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
+		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFilePrefix) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFilePrefix), "billing_file_prefix")
+		}
+		if options.Size() > 0 {
+			params.Add(options, "options")
+		}
+		removeOptions := make([]string, 0)
+		if args.RemoveOptionsBillingReportBucket {
+			removeOptions = append(removeOptions, "billing_report_bucket")
+		}
+		if args.RemoveOptionsBillingBucketAccount {
+			removeOptions = append(removeOptions, "billing_bucket_account")
+		}
+		if args.RemoveOptionsBillingFilePrefix {
+			removeOptions = append(removeOptions, "billing_file_prefix")
+		}
+		if len(removeOptions) > 0 {
+			params.Add(jsonutils.NewStringArray(removeOptions), "remove_options")
+		}
+
 		result, err := modules.Cloudaccounts.Update(s, args.ID, params)
 		if err != nil {
 			return err
@@ -337,6 +384,12 @@ func init() {
 		if len(args.OptionsBillingReportBucket) > 0 {
 			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
 		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFilePrefix) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFilePrefix), "billing_file_prefix")
+		}
 		if options.Size() > 0 {
 			params.Add(options, "options")
 		}
@@ -344,6 +397,12 @@ func init() {
 		if args.RemoveOptionsBillingReportBucket {
 			removeOptions = append(removeOptions, "billing_report_bucket")
 		}
+		if args.RemoveOptionsBillingBucketAccount {
+			removeOptions = append(removeOptions, "billing_bucket_account")
+		}
+		if args.RemoveOptionsBillingFilePrefix {
+			removeOptions = append(removeOptions, "billing_file_prefix")
+		}
 		if len(removeOptions) > 0 {
 			params.Add(jsonutils.NewStringArray(removeOptions), "remove_options")
 		}
@@ -727,4 +786,34 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type CloudaccountSyncSkusOptions struct {
+		ID       string `help:"ID or Name of cloud account"`
+		RESOURCE string `help:"Resource of skus" choices:"serversku|elasticcachesku|dbinstance_sku"`
+		Force    bool   `help:"Force sync no matter what"`
+		Provider string `help:"provider to sync"`
+		Region   string `help:"region to sync"`
+	}
+	R(&CloudaccountSyncSkusOptions{}, "cloud-account-sync-skus", "Sync skus of a cloud account", func(s *mcclient.ClientSession, args *CloudaccountSyncSkusOptions) error {
+		params := jsonutils.NewDict()
+		params.Set("resource", jsonutils.NewString(args.RESOURCE))
+		if args.Force {
+			params.Add(jsonutils.JSONTrue, "force")
+		}
+
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "cloudprovider")
+		}
+
+		if len(args.Region) > 0 {
+			params.Add(jsonutils.NewString(args.Region), "cloudregion")
+		}
+
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "sync-skus", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/cloudevents.go

@@ -24,6 +24,9 @@ func init() {
 
 	type CloudeventListOptions struct {
 		options.BaseListOptions
+
+		Since string `help:"since time"`
+		Until string `hlep:"until time"`
 	}
 	R(&CloudeventListOptions{}, "cloud-event-list", "List cloud events", func(s *mcclient.ClientSession, opts *CloudeventListOptions) error {
 		params, err := options.ListStructToParams(opts)

cmd/climc/shell/cloudregions.go

@@ -49,7 +49,7 @@ func init() {
 	type CloudregionCityListOptions struct {
 		Manager  string `help:"List objects belonging to the cloud provider"`
 		Account  string `help:"List objects belonging to the cloud account"`
-		Provider string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack|Ucloud|ZStack"`
+		Provider string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack|Ucloud|ZStack|Google|Ctyun"`
 		City     string `help:"List regions in the specified city"`
 
 		PublicCloud  *bool `help:"List objects belonging to public cloud" json:"public_cloud"`

cmd/climc/shell/contacts.go

@@ -29,11 +29,11 @@ func init() {
 	 * 操作用户的通信地址（如果用户的通信地址不存在则进行添加；如果已存在则进行修改；如果设置空则进行删除。）
 	 */
 	type ContactsUpdateOptions struct {
-		UID            string `help:"The user you wanna add contact to (Keystone User ID)"`
-		CONTACTTYPE    string `help:"The contact type email|mobile" choices:"email|mobile|dingtalk"`
-		CONTACT        string `help:"The contacts details mobile number or email address or dingtalk's userid, if set it the empty str means delete"`
-		Status         string `help:"Enabled or disabled contact status" choices:"enable|disable"`
-		UpdateDingtalk bool   `help:"if update dingtalk"`
+		UID         string   `help:"The user you wanna add contact to (Keystone User ID)"`
+		CONTACTTYPE string   `help:"The contact type email|mobile" choices:"email|mobile|dingtalk"`
+		CONTACT     string   `help:"The contacts details mobile number or email address or dingtalk's userid, if set it the empty str means delete"`
+		Status      string   `help:"Enabled or disabled contact status" choices:"enable|disable"`
+		Pull        []string `help:"pull some subcontacts(e.g., dingtalk, feishu, etc) related to mobile"`
 	}
 	R(&ContactsUpdateOptions{}, "contact-update", "Create, delete or update contact for user", func(s *mcclient.ClientSession, args *ContactsUpdateOptions) error {
 		arr := jsonutils.NewArray()
@@ -52,12 +52,14 @@ func init() {
 
 		params := jsonutils.NewDict()
 		params.Add(arr, "contacts")
-		if args.UpdateDingtalk {
-			params.Add(jsonutils.JSONTrue, "update_dingtalk")
+
+		pulls := jsonutils.NewArray()
+		for _, pull := range args.Pull {
+			pulls.Add(jsonutils.NewString(pull))
 		}
+		params.Add(pulls, "pull")
 
 		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", params)
-
 		if err != nil {
 			return err
 		}
@@ -66,6 +68,22 @@ func init() {
 		return nil
 	})
 
+	type ContactsPullOptions struct {
+		UID         string `help:"The user you wanna pull contact"`
+		CONTACTTYPE string `help:"The contact type"`
+	}
+	R(&ContactsPullOptions{}, "contact-pull", "Pull contact", func(s *mcclient.ClientSession, args *ContactsPullOptions) error {
+		params := jsonutils.NewDict()
+		params.Set("contacts", jsonutils.NewArray())
+		params.Set("pull", jsonutils.NewArray(jsonutils.NewString(args.CONTACTTYPE)))
+		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", params)
+		if err != nil {
+			return err
+		}
+		printObject(contact)
+		return nil
+	})
+
 	type ContactsDeleteOptions struct {
 		UID         string `help:"The user you wanna add contact to (Keystone User ID)"`
 		CONTACTTYPE string `help:"The contact type email|mobile|dingtalk" choices:"email|mobile|dingtalk"`

cmd/climc/shell/groups.go

@@ -79,7 +79,7 @@ func init() {
 		if err != nil {
 			return err
 		}
-		users, err := modules.Groups.GetUsers(s, grpId)
+		users, err := modules.Groups.GetUsers(s, grpId, nil)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/hosts.go

@@ -38,7 +38,7 @@ func init() {
 		Occupied  bool   `help:"show occupid host" json:"-"`
 		Enabled   bool   `help:"Show enabled host only" json:"-"`
 		Disabled  bool   `help:"Show disabled host only" json:"-"`
-		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud"`
+		HostType  string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud|google|ctyun"`
 		AnyMac    string `help:"Mac matches one of the host's interface"`
 
 		IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"`
@@ -55,6 +55,8 @@ func init() {
 
 		CdromBoot *bool `help:"filter hosts list by cdrom_boot=true|false"`
 
+		Sn string `help:"find host by sn"`
+
 		options.BaseListOptions
 	}
 	R(&HostListOptions{}, "host-list", "List hosts", func(s *mcclient.ClientSession, opts *HostListOptions) error {
@@ -76,6 +78,9 @@ func init() {
 		if len(opts.Uuid) > 0 {
 			params.Add(jsonutils.NewString(opts.Uuid), "uuid")
 		}
+		if len(opts.Sn) > 0 {
+			params.Add(jsonutils.NewString(opts.Sn), "sn")
+		}
 		result, err := modules.Hosts.List(s, params)
 		if err != nil {
 			return err

cmd/climc/shell/k8s/clusters.go

@@ -205,4 +205,70 @@ func initKubeCluster() {
 		printObject(ret)
 		return nil
 	})
+
+	R(&o.IdentOptions{}, cmdN("components-status"), "Get cluster component status", func(s *mcclient.ClientSession, args *o.IdentOptions) error {
+		ret, err := k8s.KubeClusters.GetSpecific(s, args.ID, "components-status", nil)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterComponentTypeOptions{}, cmdN("component-setting"), "Get cluster component setting", func(s *mcclient.ClientSession, args *o.ClusterComponentTypeOptions) error {
+		q := jsonutils.NewDict()
+		q.Add(jsonutils.NewString(args.TYPE), "type")
+		ret, err := k8s.KubeClusters.GetSpecific(s, args.ID, "component-setting", q)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterEnableComponentCephCSIOpt{}, cmdN("component-enable-ceph-csi"), "Enable cluster component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentCephCSIOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "enable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterDisableComponent{}, cmdN("component-disable"), "Enable cluster component", func(s *mcclient.ClientSession, args *o.ClusterDisableComponent) error {
+		params := args.Params()
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "disable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterDisableComponent{}, cmdN("component-delete"), "Delete cluster component", func(s *mcclient.ClientSession, args *o.ClusterDisableComponent) error {
+		params := args.Params()
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "delete-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterUpdateComponentCephCSIOpt{}, cmdN("component-update-ceph-csi"), "Update cluster component", func(s *mcclient.ClientSession, args *o.ClusterUpdateComponentCephCSIOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "update-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/k8s/secret.go

@@ -58,4 +58,23 @@ func initSecret() {
 			return nil
 		})
 	registryCmd.AddR(createCmd)
+
+	cephCSICmd := NewCmdNameFactory("secret-ceph-csi")
+	cephCSICreateCmd := NewCommand(
+		&o.CephCSISecretCreateOptions{},
+		cephCSICmd.Do("create"),
+		"Create ceph csi user secret",
+		func(s *mcclient.ClientSession, args *o.CephCSISecretCreateOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Secrets.Create(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+	secretCmd.AddR(cephCSICreateCmd)
 }

cmd/climc/shell/k8s/storageclass.go

@@ -15,9 +15,70 @@
 package k8s
 
 import (
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+	o "yunion.io/x/onecloud/pkg/mcclient/options/k8s"
 )
 
 func initStorageClass() {
-	initK8sClusterResource("storageclass", k8s.Storageclass)
+	cmdN := NewCmdNameFactory("storageclass")
+	scCmd := initK8sClusterResource("storageclass", k8s.Storageclass)
+
+	setDefaultCmd := NewCommand(
+		&o.ClusterResourceBaseOptions{},
+		cmdN.Do("set-default"),
+		"Set storageclass as default",
+		func(s *mcclient.ClientSession, args *o.ClusterResourceBaseOptions) error {
+			ret, err := k8s.Storageclass.PerformAction(s, args.NAME, "set-default", args.Params())
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+
+	scCmd.AddR(setDefaultCmd)
+
+	addStorageClassCephCSI(scCmd)
+}
+
+func addStorageClassCephCSI(cmd *ShellCommands) {
+	rbdN := NewCmdNameFactory("storageclass-ceph-csi-rbd")
+	rbdCreateCmd := NewCommand(
+		&o.StorageClassCephCSIRBDCreateOptions{},
+		rbdN.Do("create"),
+		"Create ceph csi rbd",
+		func(s *mcclient.ClientSession, args *o.StorageClassCephCSIRBDCreateOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Storageclass.Create(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+	testConnCmd := NewCommand(
+		&o.StorageClassCephCSIRBDTestOptions{},
+		rbdN.Do("connection-test"),
+		"Test storageclass connection",
+		func(s *mcclient.ClientSession, args *o.StorageClassCephCSIRBDTestOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Storageclass.PerformClassAction(s, "connection-test", params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+
+	cmd.AddR(rbdCreateCmd, testConnCmd)
 }

cmd/climc/shell/notification.go

@@ -29,7 +29,7 @@ func init() {
 	 */
 
 	type NotificationCreateOptions struct {
-		CONTACTTYPE string `help:"User's contacts type" choices:"email|mobile|dingtalk|webconsole"`
+		CONTACTTYPE string `help:"User's contacts type"`
 		TOPIC       string `help:"Title or topic of the notification"`
 		PRIORITY    string `help:"Priority of the notification" choices:"normal|important|fatal"`
 		MSG         string `help:"The content of the notification"`
@@ -41,7 +41,7 @@ func init() {
 		NotificationCreateOptions
 	}
 
-	R(&NotificationCreateSingleOptions{}, "notify", "Send a notification to someones", func(s *mcclient.ClientSession,
+	R(&NotificationCreateSingleOptions{}, "notify", "Send a notification to someone", func(s *mcclient.ClientSession,
 		args *NotificationCreateSingleOptions) error {
 
 		msg := notify.SNotifyMessage{}

cmd/climc/shell/notify.go

@@ -34,7 +34,24 @@ func init() {
 		}
 		body := jsonutils.NewDict()
 		body.Add(tmp, args.CONTACTTYPE)
-		modules.Configs.Create(s, body)
+		ret, err := modules.Configs.Create(s, body)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+	R(&ConfigCreate2Options{}, "notify-config-validate", "config validate", func(s *mcclient.ClientSession,
+		args *ConfigCreate2Options) error {
+		tmp := jsonutils.NewDict()
+		for i := 0; i+1 < len(args.CONFIGS); i += 2 {
+			tmp.Add(jsonutils.NewString(args.CONFIGS[i+1]), args.CONFIGS[i])
+		}
+		ret, err := modules.Configs.PerformAction(s, args.CONTACTTYPE, "validate", tmp)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
 		return nil
 	})
 

cmd/climc/shell/quotas.go

@@ -24,7 +24,7 @@ import (
 )
 
 type ComputeQuotaKeys struct {
-	RegionQuotaKeys
+	ZoneQuotaKeys
 
 	Hypervisor string `help:"hypervisor" choices:"kvm|baremetal"`
 }
@@ -53,8 +53,8 @@ type RegionQuotaOptions struct {
 
 	Port  int64 `help:"Internal NIC count" json:"port,omitzero"`
 	Eport int64 `help:"External NIC count" json:"eport,omitzero"`
-	Bw    int64 `help:"Internal bandwidth in Mbps" json:"bw,omitzero"`
-	Ebw   int64 `help:"External bandwidth in Mbps" json:"ebw,omitzero"`
+	// Bw    int64 `help:"Internal bandwidth in Mbps" json:"bw,omitzero"`
+	// Ebw   int64 `help:"External bandwidth in Mbps" json:"ebw,omitzero"`
 
 	Eip int64 `help:"Elastic IP count" json:"eip,omitzero"`
 
@@ -110,6 +110,7 @@ func init() {
 		Project string `help:"Tenant name or ID" json:"tenant"`
 		Domain  string `help:"Domain name or ID" json:"domain"`
 		Refresh bool   `help:"refresh" json:"refresh,omitfalse"`
+		Primary bool   `help:"get primary quota ONLY" json:"primary,omitfalse"`
 	}
 	R(&QuotaOptions{}, "quota", "Show quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
 		params := jsonutils.Marshal(args)
@@ -239,6 +240,8 @@ func init() {
 
 	type QuotaListOptions struct {
 		ProjectDomain string `help:"domain name or ID to query project quotas"`
+		Refresh       bool   `help:"refresh" json:"refresh,omitfalse"`
+		Primary       bool   `help:"list primary quota ONLY" json:"primary,omitfalse"`
 	}
 	R(&QuotaListOptions{}, "quota-list", "List quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
 		params := jsonutils.Marshal(args)

cmd/climc/shell/reservedips.go

@@ -28,6 +28,7 @@ func init() {
 		NOTES    string   `help:"Why reserve this IP"`
 		IPS      []string `help:"IPs to reserve"`
 		Duration string   `help:"reservation duration, e.g. 1I, 1H, 2M"`
+		Status   string   `help:"ip status"`
 	}
 	R(&NetworkReserveIPOptions{}, "network-reserve-ip", "Reserve an IP address from pool", func(s *mcclient.ClientSession, args *NetworkReserveIPOptions) error {
 		params := jsonutils.NewDict()
@@ -36,6 +37,9 @@ func init() {
 		if len(args.Duration) > 0 {
 			params.Add(jsonutils.NewString(args.Duration), "duration")
 		}
+		if len(args.Status) > 0 {
+			params.Add(jsonutils.NewString(args.Status), "status")
+		}
 		net, err := modules.Networks.PerformAction(s, args.NETWORK, "reserve-ip", params)
 		if err != nil {
 			return err

cmd/climc/shell/servers.go

@@ -178,6 +178,19 @@ func init() {
 		},
 	)
 
+	R(&options.ServerCreateOptions{}, "server-check-create-data", "Check create server data", func(s *mcclient.ClientSession, opts *options.ServerCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		server, err := modules.Servers.PerformClassAction(s, "check-create-data", params.JSON(params))
+		if err != nil {
+			return err
+		}
+		printObject(server)
+		return nil
+	})
+
 	R(&options.ServerCreateOptions{}, "server-create", "Create a server", func(s *mcclient.ClientSession, opts *options.ServerCreateOptions) error {
 		params, err := opts.Params()
 		if err != nil {

cmd/climc/shell/usages.go

@@ -22,9 +22,9 @@ import (
 )
 
 type GeneralUsageOptions struct {
-	HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack"`
-	Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack"`
-	Brand    []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack"`
+	HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack|google|ctyun"`
+	Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|Google|Ctyun"`
+	Brand    []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack|Google|Ctyun"`
 	Project  string   `help:"show usage of specified project"`
 
 	ProjectDomain string `help:"show usage of specified domain"`

cmd/esxi-agent/main.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/esxi/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
+
+func main() {
+	defer atexit.Handle()
+
+	service.New().StartService()
+}

go.mod

@@ -3,6 +3,7 @@ module yunion.io/x/onecloud
 go 1.12
 
 require (
+	cloud.google.com/go v0.38.0
 	github.com/360EntSecGroup-Skylar/excelize v1.4.0
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible
 	github.com/Azure/go-autorest v10.15.5+incompatible
@@ -109,16 +110,20 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
 	github.com/ugorji/go v0.0.0-20161130061742-9c7f9b7a2bc3 // indirect
+	github.com/vishvananda/netlink v1.0.0
+	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible
 	github.com/vmware/govmomi v0.19.0
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
 	golang.org/x/net v0.0.0-20191007182048-72f939374954
-	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191008105621-543471e840be
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
+	google.golang.org/api v0.13.0 // indirect
+	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
 	google.golang.org/grpc v1.23.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
@@ -132,12 +137,12 @@ require (
 	k8s.io/client-go v9.0.0+incompatible
 	k8s.io/klog v0.1.0 // indirect
 	k8s.io/kubernetes v1.12.3
-	yunion.io/x/executor v0.0.0-20191202093616-92e2e6119257
-	yunion.io/x/jsonutils v0.0.0-20191220095028-7fef3b75f64d
+	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
+	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20191221094533-5097f12f41ac
+	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20191226074733-6eb73845bfb7
+	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 )
 

go.sum

@@ -3,6 +3,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
 cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
+cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 github.com/360EntSecGroup-Skylar/excelize v1.4.0 h1:43rak9uafmwSJpXfFO1heKQph8tP3nlfWJWFQQtW1R0=
 github.com/360EntSecGroup-Skylar/excelize v1.4.0/go.mod h1:R8KYLmGns0vDPe6/HyphW0mzW+MFexlGDafU0ykVEnU=
 github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -222,7 +224,10 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
 github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googollee/go-engine.io v0.0.0-20180829091931-e2f255711dcb h1:n22Aukg/TjoypWc37dbKIpCsz0VMFPD36HQk1WKvg3A=
@@ -256,7 +261,10 @@ github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead h1:Y9WOGZY2nw5ksbEf5AIpk+vK52Tdg/VN/rHFRfEeeGQ=
 github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
@@ -481,6 +489,10 @@ github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c/go.mod h1:M/I710pXKQ
 github.com/ugorji/go v0.0.0-20161130061742-9c7f9b7a2bc3 h1:KfqgRfYvGRyKOQUx8GphSC8D6vVoBndzco3rESP8I4k=
 github.com/ugorji/go v0.0.0-20161130061742-9c7f9b7a2bc3/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
+github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmware/govmomi v0.19.0 h1:CR6tEByWCPOnRoRyhLzuHaU+6o2ybF3qufNRWS/MGrY=
@@ -499,7 +511,10 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
+go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -524,6 +539,7 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -537,6 +553,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -548,6 +565,8 @@ golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -572,6 +591,7 @@ golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190830023255-19e00faab6ad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -595,23 +615,34 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.zx2c4.com/wireguard v0.0.20190908 h1:SUoXDdwSMtomLdvke+zz83/u9tNvl4hHmcTIWp38tow=
 golang.zx2c4.com/wireguard v0.0.20190908/go.mod h1:LhfXh5z6bLC2lW2ve6BzYZFwnnsXK3OQjySR0Yh2dO8=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987 h1:26OAgqBTufVr8WKonCEhhjO1oKsYhHv0iM5Dg92G1TM=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987/go.mod h1:7hq1rEDsx7/FWl8IEEnfH2Xhs6M2MNnjUfN0PeI8Rm0=
+google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.13.0 h1:Q3Ui3V3/CVinFWFiW39Iw0kMuVrRzYX0wN6OPFp0lTA=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo=
 google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
@@ -645,6 +676,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 k8s.io/api v0.0.0-20181004124137-fd83cbc87e76 h1:cGc6jt7tNK7a2WfgNKjxjoU/UXXr9Q7JTqvCupZ+6+Y=
 k8s.io/api v0.0.0-20181004124137-fd83cbc87e76/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
@@ -658,12 +690,12 @@ k8s.io/kubernetes v1.12.3 h1:5GPfYyyBylqcZUqL+ApYpYTm2IYjH56JUewYC0GbetU=
 k8s.io/kubernetes v1.12.3/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-yunion.io/x/executor v0.0.0-20191202093616-92e2e6119257 h1:LZ6eC1uoLDAp7NQlaCy+DKnF31Mgd3LDKlIithziEmY=
-yunion.io/x/executor v0.0.0-20191202093616-92e2e6119257/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
+yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051 h1:vtZw2iwGrsARNSwRTREGjmr2BWPdxbmXVkb3kI1qu28=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20191220095028-7fef3b75f64d h1:58hs2hyO2DgeVQ5i2fAKjg/ooPH0q4+ALEnqtxjn7DA=
-yunion.io/x/jsonutils v0.0.0-20191220095028-7fef3b75f64d/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
+yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda h1:wSwQj3MDGchGYM2RAo1riYhrTz4apH+5XyBnchuDi84=
+yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b h1:Z9z+7iegu0HXuL+S8taVWRd1P4b9JJOgPXIeoqYrj7c=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
@@ -671,11 +703,12 @@ yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRiE7QcPG8eTp0GmsHbg=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20191221094533-5097f12f41ac h1:upUF6LBhCKx9c/OUzSUwfv50oCtU4LLmO+UtnKEQsg8=
-yunion.io/x/pkg v0.0.0-20191221094533-5097f12f41ac/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84=
+yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20191226074733-6eb73845bfb7 h1:nK047S9fIjIBkB9Yyo2e6itUA7TpmFsd7/a6mGKjNU0=
-yunion.io/x/sqlchemy v0.0.0-20191226074733-6eb73845bfb7/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
+yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apigateway/handler/misc.go

@@ -285,6 +285,12 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 		domain := row[2]
 		allowWebConsole := strings.ToLower(row[3])
 
+		// 忽略空白行
+		rowStr := strings.Join(row, "")
+		if len(strings.TrimSpace(rowStr)) == 0 {
+			continue
+		}
+
 		if len(name) == 0 {
 			e := httperrors.NewClientError("row %d name is empty", rowIdx)
 			httperrors.JsonClientError(w, e)

pkg/apigateway/handler/rpc.go

@@ -117,21 +117,24 @@ func rpcHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 		v, ok := retobj.Interface().(jsonutils.JSONObject)
 		if ok {
 			appsrv.SendJSON(w, v)
+			return
 		}
 
 		v2, ok := retobj.Interface().([]modulebase.SubmitResult)
 		if ok {
 			w.WriteHeader(207)
 			appsrv.SendJSON(w, modulebase.SubmitResults2JSON(v2))
-		} else {
-			httperrors.BadGatewayError(w, "recv invalid data")
+			return
 		}
+
+		httperrors.BadGatewayError(w, "recv invalid data")
 	} else {
 		v, ok := reterr.Interface().(*httputils.JSONClientError)
 		if ok {
 			httperrors.JsonClientError(w, v)
-		} else {
-			httperrors.BadGatewayError(w, fmt.Sprintf("%s", reterr.Interface()))
+			return
 		}
+
+		httperrors.BadGatewayError(w, fmt.Sprintf("%s", reterr.Interface()))
 	}
 }

pkg/apigateway/options/options.go

@@ -29,7 +29,7 @@ type GatewayOptions struct {
 
 	SqlitePath string `help:"sqlite db path" default:"/etc/yunion/data/yunionapi.db"`
 
-	common_options.CommonOptions
+	common_options.CommonOptions `"request_worker_count->default":"32"`
 }
 
 var (
@@ -40,9 +40,10 @@ func OnOptionsChange(oldO, newO interface{}) bool {
 	oldOpts := oldO.(*GatewayOptions)
 	newOpts := newO.(*GatewayOptions)
 
+	changed := false
 	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
-		return true
+		changed = true
 	}
 
-	return false
+	return changed
 }

pkg/apis/cloudevent/cloudevent.go

@@ -15,15 +15,24 @@
 package cloudevent
 
 import (
+	"time"
+
 	"yunion.io/x/onecloud/pkg/apis"
 )
 
 type CloudeventListInput struct {
 	apis.BaseListInput
 
-	apis.SharableVirtualResourceListInput
-
-	Cloudprovider string `json:"cloudprovider"`
-
+	// 根据平台过滤
 	Providers []string `json:"providers"`
+	// 根据品牌过滤
+	Brands []string `json:"brands"`
+	// swagger:ignore
+	// Deprecated
+	Brand []string `json:"brand" deprecated-by:"brands"`
+
+	// 操作日志起始日期
+	Since time.Time `json:"since"`
+	// 操作日志截止日期
+	Until time.Time `json:"until"`
 }

pkg/apis/compute/api.go

@@ -164,6 +164,8 @@ type ServerCreateInput struct {
 	UserData  string            `json:"user_data"`
 	Metadata  map[string]string `json:"__meta__"`
 
+	// swagger:ignore
+	Keypair   string `json:"keypair" deprecated-by:"keypair_id"`
 	KeypairId string `json:"keypair_id"`
 	Password  string `json:"password"`
 	Cdrom     string `json:"cdrom"`

pkg/apis/compute/bucket.go

@@ -52,6 +52,7 @@ type BucketCreateInput struct {
 type BucketDetail struct {
 	apis.Meta
 	SBucket
+	CloudproviderDetails
 }
 
 type BucketObjectsActionInput struct {

pkg/apis/compute/cloudaccount.go

@@ -27,6 +27,7 @@ type CloudaccountCreateInput struct {
 	apis.EnabledStatusStandaloneResourceCreateInput
 
 	Provider            string
+	AccountId           string
 	Brand               string
 	IsPublicCloud       bool
 	IsOnPremise         bool

pkg/apis/compute/cloudregions_const.go

@@ -36,6 +36,7 @@ const (
 	CITY_TAIPEI        = "Taipei"      //台北市
 	CITY_KAOHSIUNG     = "Kaohsiung"   //高雄市
 	CITY_CHENG_DU      = "Chengdu"     //成都
+	CITY_HE_YUAN       = "HeYuan"      //河源
 	CITY_CHONG_QING    = "Chongqing"   //重庆
 	CITY_LAN_ZHOU      = "Lanzhou"     //兰州
 	CITY_TAI_YUAN      = "Taiyuan"     //太原
@@ -76,6 +77,11 @@ const (
 
 	//瑞士
 	CITY_ZURICH = "Zurich" //苏黎世
+	CITY_GENEVA = "Geneva" //日内瓦
+
+	// 挪威
+	CITY_STAVANGER = "Stavanger" //斯塔万格
+	CITY_OSLO      = "Oslo"      // 奥斯陆
 
 	// 马来西亚
 	CITY_KUALA_LUMPUR = "Kuala Lumpur" //吉隆坡
@@ -104,6 +110,7 @@ const (
 	CITY_IOWA           = "Iowa"           //爱荷华
 	CITY_US_GOV_WEST    = "us-gov-west"    //???
 	CITY_SOUTH_CAROLINA = "South Carolina" //南卡罗来纳州
+	CITY_SALT_LAKE_CITY = "Salt Lake City" //盐湖城
 
 	// 英国
 	CITY_LONDON      = "London"      //伦敦
@@ -114,7 +121,8 @@ const (
 	CITY_DUBAI = "Dubai" //迪拜
 
 	// 德国
-	CITY_FRANKFURT = "Frankfurt" //法兰克福
+	CITY_FRANKFURT   = "Frankfurt"   // 法兰克福
+	CITY_DELMENHORST = "Delmenhorst" // 代尔门霍斯特
 
 	// 韩国
 	CITY_SEOUL = "Seoul" //首尔
@@ -185,4 +193,5 @@ const (
 	COUNTRY_CODE_NG = "NG" //尼日利亚
 	COUNTRY_CODE_VN = "VN" //越南
 	COUNTRY_CODE_CH = "CH" //瑞士
+	COUNTRY_CODE_NO = "NO" //挪威
 )

pkg/apis/compute/dbinstance_const.go

@@ -108,10 +108,10 @@ const (
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_CLOUD_SSD  = "cloud_ssd"  //SSD云盘
 
 	//华为云存储类型
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SSD  = "SSD"  //超高IO云硬盘
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SAS  = "SAS"  //高IO云硬盘
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SATA = "SATA" //普通IO云硬盘
-
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_ULTRAHIGH    = "ULTRAHIGH" //超高IO云硬盘
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_ULTRAHIGHPRO = "ULTRAHIGHPRO"
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_COMMON       = "COMMON"
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_HIGH         = "HIGH"
 )
 
 var (

pkg/apis/compute/elasticips_const.go

@@ -39,10 +39,6 @@ const (
 	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
 	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
 	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
-
-	// EIP associate resource type
-	EIP_ASSOCIATE_TYPE_VM  = "server"
-	EIP_ASSOCIATE_TYPE_ELB = "elb"
 )
 
 var (

pkg/apis/compute/guest_const.go

@@ -150,7 +150,13 @@ const (
 	HYPERVISOR_DEFAULT = HYPERVISOR_KVM
 )
 
+const (
+	CPU_MODE_QEMU = "qemu"
+	CPU_MODE_HOST = "host"
+)
+
 var VM_RUNNING_STATUS = []string{VM_START_START, VM_STARTING, VM_RUNNING, VM_BLOCK_STREAM}
+
 var VM_CREATING_STATUS = []string{VM_CREATE_NETWORK, VM_CREATE_DISK, VM_START_DEPLOY, VM_DEPLOYING}
 
 var HYPERVISORS = []string{

pkg/apis/compute/guests.go

@@ -65,6 +65,8 @@ type ServerListInput struct {
 	ResourceType string `json:"resource_type"`
 	// Billing type choices:"postpaid|prepaid"
 	BillingType string `json:"billing_type"`
+	// 根据宿主机 SN 过滤
+	HostSn string `json:"host_sn"`
 }
 
 type ServerRebuildRootInput struct {

pkg/apis/compute/guesttemplate.go

@@ -55,6 +55,7 @@ type GuesttemplateConfigInfo struct {
 	Secgroup             string                 `json:"secgroup"`
 	IsolatedDeviceConfig []IsolatedDeviceConfig `json:"isolated_device_config"`
 	Image                string                 `json:"image"`
+	ResetPassword        bool                   `json:"reset_password"`
 }
 
 type GuesttemplateDisk struct {

pkg/apis/compute/loadbalancer_const.go

@@ -275,6 +275,38 @@ var LB_HEALTH_CHECK_HTTP_CODES = choices.NewChoices(
 	LB_HEALTH_CHECK_HTTP_CODE_5xx,
 )
 
+const (
+	LB_REDIRECT_OFF = "off"
+	LB_REDIRECT_RAW = "raw"
+)
+
+var LB_REDIRECT_TYPES = choices.NewChoices(
+	LB_REDIRECT_OFF,
+	LB_REDIRECT_RAW,
+)
+
+const (
+	LB_REDIRECT_CODE_301 = int64(301) // Moved Permanently
+	LB_REDIRECT_CODE_302 = int64(302) // Found
+	LB_REDIRECT_CODE_307 = int64(307) // Temporary Redirect
+)
+
+var LB_REDIRECT_CODES = []int64{
+	LB_REDIRECT_CODE_301,
+	LB_REDIRECT_CODE_302,
+	LB_REDIRECT_CODE_307,
+}
+
+const (
+	LB_REDIRECT_SCHEME_HTTP  = "http"
+	LB_REDIRECT_SCHEME_HTTPS = "https"
+)
+
+var LB_REDIRECT_SCHEMES = choices.NewChoices(
+	LB_REDIRECT_SCHEME_HTTP,
+	LB_REDIRECT_SCHEME_HTTPS,
+)
+
 const (
 	LB_BOOL_ON  = "on"
 	LB_BOOL_OFF = "off"

pkg/apis/compute/network_const.go

@@ -59,6 +59,7 @@ var (
 
 	REGIONAL_NETWORK_PROVIDERS = []string{
 		CLOUD_PROVIDER_HUAWEI,
+		CLOUD_PROVIDER_CTYUN,
 		CLOUD_PROVIDER_UCLOUD,
 		CLOUD_PROVIDER_GOOGLE,
 	}

pkg/apis/compute/routetable_const.go

@@ -0,0 +1,23 @@
+package compute
+
+const (
+	ROUTE_TABLE_TYPE_VPC = "VPC" // VPC路由器
+	ROUTE_TABLE_TYPE_VBR = "VBR" // 边界路由器
+)
+
+const (
+	ROUTE_ENTRY_TYPE_CUSTOM = "Custom" // 自定义路由
+	ROUTE_ENTRY_TYPE_SYSTEM = "System" // 系统路由
+)
+
+const (
+	Next_HOP_TYPE_INSTANCE        = "Instance"              // ECS实例。
+	Next_HOP_TYPE_HAVIP           = "HaVip"                 // 高可用虚拟IP。
+	Next_HOP_TYPE_VPN             = "VpnGateway"            // VPN网关。
+	Next_HOP_TYPE_NAT             = "NatGateway"            // NAT网关。
+	Next_HOP_TYPE_NETWORK         = "NetworkInterface"      // 辅助弹性网卡。
+	Next_HOP_TYPE_ROUTER          = "RouterInterface"       // 路由器接口。
+	Next_HOP_TYPE_IPV6            = "IPv6Gateway"           // IPv6网关。
+	Next_HOP_TYPE_INTERNET        = "InternetGateway"       // Internet网关。
+	Next_HOP_TYPE_EGRESS_INTERNET = "EgressInternetGateway" // egress only Internet网关。
+)

pkg/apis/identity/zz_generated.model.go

@@ -34,7 +34,7 @@ type SAssignment struct {
 type SConfigOption struct {
 	apis.SResourceBase
 	ResType string      `json:"res_type"`
-	ResId   string      `json:"res_id"`
+	ResId   string      `json:"domain_id"`
 	Group   string      `json:"group"`
 	Option  string      `json:"option"`
 	Value   interface{} `json:"value"`
@@ -91,6 +91,12 @@ type SFederatedUser struct {
 	DisplayName string `json:"display_name"`
 }
 
+// SGroup is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SGroup.
+type SGroup struct {
+	SIdentityBaseResource
+	Displayname string `json:"displayname"`
+}
+
 // SIdentityBaseResource is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SIdentityBaseResource.
 type SIdentityBaseResource struct {
 	apis.SStandaloneResourceBase
@@ -101,13 +107,14 @@ type SIdentityBaseResource struct {
 // SIdentityProvider is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SIdentityProvider.
 type SIdentityProvider struct {
 	apis.SEnabledStatusStandaloneResourceBase
-	Driver              string    `json:"driver"`
-	Template            string    `json:"template"`
-	TargetDomainId      string    `json:"target_domain_id"`
-	AutoCreateProject   *bool     `json:"auto_create_project,omitempty"`
-	ErrorCount          int       `json:"error_count"`
-	SyncStatus          string    `json:"sync_status"`
-	LastSync            time.Time `json:"last_sync"`
+	Driver            string    `json:"driver"`
+	Template          string    `json:"template"`
+	TargetDomainId    string    `json:"target_domain_id"`
+	AutoCreateProject *bool     `json:"auto_create_project,omitempty"`
+	ErrorCount        int       `json:"error_count"`
+	SyncStatus        string    `json:"sync_status"`
+	LastSync          time.Time `json:"last_sync"`
+	// = Column(DateTime, nullable=True)
 	LastSyncEndAt       time.Time `json:"last_sync_end_at"`
 	SyncIntervalSeconds int       `json:"sync_interval_seconds"`
 }
@@ -116,8 +123,8 @@ type SIdentityProvider struct {
 type SIdmapping struct {
 	apis.SResourceBase
 	PublicId    string `json:"public_id"`
-	IdpId       string `json:"idp_id"`
-	IdpEntityId string `json:"idp_entity_id"`
+	IdpId       string `json:"domain_id"`
+	IdpEntityId string `json:"local_id"`
 	EntityType  string `json:"entity_type"`
 }
 
@@ -153,6 +160,13 @@ type SPolicy struct {
 	Blob interface{} `json:"blob"`
 }
 
+// SProject is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SProject.
+type SProject struct {
+	SIdentityBaseResource
+	ParentId string `json:"parent_id"`
+	IsDomain *bool  `json:"is_domain,omitempty"`
+}
+
 // SRegion is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SRegion.
 type SRegion struct {
 	apis.SStandaloneResourceBase
@@ -174,6 +188,22 @@ type SService struct {
 	Extra   interface{} `json:"extra"`
 }
 
+// SUser is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SUser.
+type SUser struct {
+	SEnabledIdentityBaseResource
+	Email           string    `json:"email"`
+	Mobile          string    `json:"mobile"`
+	Displayname     string    `json:"displayname"`
+	LastActiveAt    time.Time `json:"last_active_at"`
+	LastLoginIp     string    `json:"last_login_ip"`
+	LastLoginSource string    `json:"last_login_source"`
+	IsSystemAccount *bool     `json:"is_system_account,omitempty"`
+	// deprecated
+	DefaultProjectId string `json:"default_project_id"`
+	AllowWebConsole  *bool  `json:"allow_web_console,omitempty"`
+	EnableMfa        *bool  `json:"enable_mfa,omitempty"`
+}
+
 // SUsergroupMembership is an autogenerated struct via yunion.io/x/onecloud/pkg/keystone/models.SUsergroupMembership.
 type SUsergroupMembership struct {
 	apis.SResourceBase

pkg/apis/image/zz_generated.model.go

@@ -21,7 +21,7 @@ import (
 // SGuestImage is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SGuestImage.
 type SGuestImage struct {
 	apis.SSharableVirtualResourceBase
-	Protected *bool `json:"omitempty,protected"`
+	Protected *bool `json:"protected,omitempty"`
 }
 
 // SGuestImageJoint is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SGuestImageJoint.
@@ -34,19 +34,22 @@ type SGuestImageJoint struct {
 // SImage is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImage.
 type SImage struct {
 	apis.SSharableVirtualResourceBase
-	Size         int64  `json:"size"`
-	Location     string `json:"location"`
-	DiskFormat   string `json:"disk_format"`
+	Size int64 `json:"size"`
+	// VirtualSize int64  `nullable:"true" list:"user" create:"optional"`
+	Location   string `json:"location"`
+	DiskFormat string `json:"disk_format"`
+	// Column(VARCHAR(32, charset='ascii'), nullable=False, default='qcow2')
 	Checksum     string `json:"checksum"`
 	FastHash     string `json:"fast_hash"`
 	Owner        string `json:"owner"`
-	MinDiskMB    int32  `json:"min_disk_mb"`
-	MinRamMB     int32  `json:"min_ram_mb"`
-	Protected    *bool  `json:"omitempty,protected"`
+	MinDiskMB    int32  `json:"min_disk"`
+	MinRamMB     int32  `json:"min_ram"`
+	Protected    *bool  `json:"protected,omitempty"`
 	IsStandard   *bool  `json:"is_standard,omitempty"`
 	IsGuestImage *bool  `json:"is_guest_image,omitempty"`
 	IsData       *bool  `json:"is_data,omitempty"`
-	OssChecksum  string `json:"oss_checksum"`
+	// image copy from url, save origin checksum before probe
+	OssChecksum string `json:"oss_checksum"`
 }
 
 // SImagePeripheral is an autogenerated struct via yunion.io/x/onecloud/pkg/image/models.SImagePeripheral.

pkg/apis/scheduler/api.go

@@ -77,6 +77,9 @@ type ScheduleInput struct {
 	LiveMigrate  bool   `json:"live_migrate"`
 	CpuDesc      string `json:"cpu_desc"`
 	CpuMicrocode string `json:"cpu_microcode"`
+	CpuMode      string `json:"cpu_mode"`
+
+	PendingUsages []jsonutils.JSONObject
 }
 
 func (input ScheduleInput) ToConditionInput() *jsonutils.JSONDict {
@@ -91,6 +94,17 @@ type CandidateDisk struct {
 	StorageIds []string `json:"storage_ids"`
 }
 
+type CandidateDiskV2 struct {
+	Index    int                 `json:"index"`
+	Storages []*CandidateStorage `json:"storages"`
+}
+
+type CandidateStorage struct {
+	Id           string
+	Name         string
+	FreeCapacity int64
+}
+
 type CandidateNet struct {
 	Index      int      `json:"index"`
 	NetworkIds []string `json:"network_ids"`

pkg/apis/zz_generated.model.go

@@ -57,7 +57,7 @@ type SKeystoneCacheObject struct {
 // SProjectizedResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SProjectizedResourceBase.
 type SProjectizedResourceBase struct {
 	SDomainizedResourceBase
-	ProjectId string `json:"project_id"`
+	ProjectId string `json:"tenant_id"`
 }
 
 // SResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SResourceBase.
@@ -72,7 +72,7 @@ type SResourceBase struct {
 // SScopedResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SScopedResourceBase.
 type SScopedResourceBase struct {
 	DomainId  string `json:"domain_id"`
-	ProjectId string `json:"project_id"`
+	ProjectId string `json:"tenant_id"`
 }
 
 // SSharableBaseResource is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db.SSharableBaseResource.

pkg/appsrv/appsrv.go

@@ -268,7 +268,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 		hand, ok := handler.(*SHandlerInfo)
 		if ok {
 			fw := newResponseWriterChannel(w)
-			worker := make(chan *SWorker)
+			currentWorker := make(chan *SWorker, 1) // make it a buffered channel
 			to := hand.FetchProcessTimeout(r)
 			if to == 0 {
 				to = app.processTimeout
@@ -320,13 +320,13 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 					} // otherwise, the task has been timeout
 					fw.closeChannels()
 				},
-				worker,
+				currentWorker,
 				func(err error) {
 					httperrors.InternalServerError(&fw, "Internal server error: %s", err)
 					fw.closeChannels()
 				},
 			)
-			runErr := fw.wait(ctx, worker)
+			runErr := fw.wait(ctx, currentWorker)
 			if runErr != nil {
 				switch runErr.(type) {
 				case *httputils.JSONClientError:

pkg/appsrv/response.go

@@ -100,7 +100,12 @@ func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWork
 	stop := false
 	for !stop {
 		select {
-		case worker = <-workerChan:
+		case curWorker, more := <-workerChan:
+			if more {
+				worker = curWorker
+			} else {
+				// ignore, worker is responsible for close the channel
+			}
 		case <-ctx.Done():
 			// ctx deadline reached, timeout
 			if worker != nil {

pkg/appsrv/workers.go

@@ -83,6 +83,9 @@ func (worker *SWorker) run() {
 			task := req.(*sWorkerTask)
 			if task.worker != nil {
 				task.worker <- worker
+				// worker channel is buffered
+				// close the worker channel
+				close(task.worker)
 			}
 			execCallback(task)
 		} else {

pkg/baremetal/handler/middleware.go

@@ -156,11 +156,15 @@ func bmRegisterMiddleware(h bmRegisterFunc) appsrv.FilterHandler {
 			newCtx.ResponseError(httperrors.NewMissingParameterError("hostname"))
 			return
 		}
-		remoteIp, _, err := net.SplitHostPort(r.RemoteAddr)
-		if err != nil {
-			newCtx.ResponseError(httperrors.NewInternalServerError("Parse ip error %s", err))
-			return
+		remoteIp, _ := newCtx.Data().GetString("ssh_ip")
+		if len(remoteIp) == 0 {
+			remoteIp, _, err = net.SplitHostPort(r.RemoteAddr)
+			if err != nil {
+				newCtx.ResponseError(httperrors.NewInternalServerError("Parse ip error %s", err))
+				return
+			}
 		}
+
 		sshPort, err := newCtx.Data().Int("ssh_port")
 		if err != nil {
 			newCtx.ResponseError(httperrors.NewMissingParameterError("ssh_port"))

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -688,7 +688,7 @@ func (adapter *MegaRaidAdaptor) storcliIsJBODEnabled() bool {
 		line = strings.ToLower(line)
 		if strings.HasPrefix(line, "jbod") {
 			data := strings.Split(line, " ")
-			if strings.TrimSpace(data[1]) == "on" {
+			if strings.TrimSpace(data[len(data)-1]) == "on" {
 				return true
 			}
 			return false

pkg/cloudcommon/agent/agent.go

@@ -24,6 +24,8 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/version"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/agent/iagent"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -31,19 +33,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
-type IAgent interface {
-	GetAgentType() string
-	GetAccessIP() (net.IP, error)
-	GetListenIP() (net.IP, error)
-	GetPort() int
-	GetEnableSsl() bool
-	GetZoneName() string
-	GetAdminSession() *mcclient.ClientSession
-	TuneSystem() error
-	StartService() error
-	StopService() error
-}
-
 type SZoneInfo struct {
 	Name string `json:"name"`
 	Id   string `json:"id"`
@@ -80,11 +69,11 @@ func getIfaceIPs(iface *net.Interface) ([]net.IP, error) {
 	return ips, nil
 }
 
-func (agent *SBaseAgent) IAgent() IAgent {
-	return agent.GetVirtualObject().(IAgent)
+func (agent *SBaseAgent) IAgent() iagent.IAgent {
+	return agent.GetVirtualObject().(iagent.IAgent)
 }
 
-func (agent *SBaseAgent) Init(iagent IAgent, ifname string, cachePath string) error {
+func (agent *SBaseAgent) Init(iagent iagent.IAgent, ifname string, cachePath string) error {
 	iface, err := net.InterfaceByName(ifname)
 	if err != nil {
 		return err
@@ -246,7 +235,9 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
 	if err != nil {
 		return err
 	}
-	params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
+	if agent.IAgent().GetAgentType() != string(api.AgentTypeEsxi) {
+		params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
+	}
 	params.Add(jsonutils.NewString(agent.IAgent().GetAgentType()), "agent_type")
 	ret, err := modules.Baremetalagents.List(session, params)
 	if err != nil {

pkg/cloudcommon/agent/iagent/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iagent // import "yunion.io/x/onecloud/pkg/cloudcommon/agent/iagent"

pkg/cloudcommon/agent/iagent/interface.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iagent
+
+import (
+	"net"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type IAgent interface {
+	GetAgentType() string
+	GetAccessIP() (net.IP, error)
+	GetListenIP() (net.IP, error)
+	GetPort() int
+	GetEnableSsl() bool
+	GetZoneName() string
+	GetAdminSession() *mcclient.ClientSession
+	TuneSystem() error
+	StartService() error
+	StopService() error
+}

pkg/cloudcommon/app/app.go

@@ -28,6 +28,7 @@ import (
 
 func InitApp(options *common_options.BaseOptions, dbAccess bool) *appsrv.Application {
 	// cache := appsrv.NewCache(options.AuthTokenCacheSize)
+	log.Infof("RequestWorkerCount: %d", options.RequestWorkerCount)
 	app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess)
 	app.CORSAllowHosts(options.CorsHosts)
 

pkg/cloudcommon/consts/consts.go

@@ -16,6 +16,8 @@ package consts
 
 import (
 	"time"
+
+	"yunion.io/x/log"
 )
 
 var (
@@ -28,6 +30,9 @@ var (
 	tenantCacheExpireSeconds = 900
 
 	nonDefaultDomainProjects = false
+
+	defaultPagingLimit int64 = 2048
+	maxPagingLimit     int64 = 2048
 )
 
 func SetRegion(region string) {
@@ -55,9 +60,18 @@ func GetTenantCacheExpireSeconds() time.Duration {
 }
 
 func SetNonDefaultDomainProjects(val bool) {
+	log.Infof("set non_default_domain_projects to %v", val)
 	nonDefaultDomainProjects = val
 }
 
 func GetNonDefaultDomainProjects() bool {
 	return nonDefaultDomainProjects
 }
+
+func GetDefaultPagingLimit() int64 {
+	return defaultPagingLimit
+}
+
+func GetMaxPagingLimit() int64 {
+	return maxPagingLimit
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/filterclause"
 	"yunion.io/x/pkg/utils"
@@ -38,15 +39,10 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
-	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
-var (
-	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
-)
-
 type DBModelDispatcher struct {
 	modelManager IModelManager
 }
@@ -448,7 +444,7 @@ func fetchContextObject(manager IModelManager, ctx context.Context, userCred mcc
 
 func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modulebase.ListResult, error) {
 	var err error
-	var maxLimit int64 = 2048
+	var maxLimit int64 = consts.GetMaxPagingLimit()
 	limit, _ := query.Int("limit")
 	offset, _ := query.Int("offset")
 	pagingMarker, _ := query.GetString("paging_marker")
@@ -479,6 +475,9 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	var totalCnt int
 	pagingConf := manager.GetPagingConfig()
 	if pagingConf == nil {
+		if limit == 0 {
+			limit = consts.GetDefaultPagingLimit()
+		}
 		totalCnt, err = q.CountWithError()
 		if err != nil {
 			return nil, err
@@ -503,6 +502,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		limit = exportLimit
 	}
 
+	// orders defined in pagingConf should have the highest priority
+	if pagingConf != nil {
+		for _, f := range pagingConf.MarkerFields {
+			if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+				q = q.Asc(f)
+			} else {
+				q = q.Desc(f)
+			}
+		}
+	}
+
 	var primaryCol sqlchemy.IColumnSpec
 	primaryCols := manager.TableSpec().PrimaryColumns()
 	if len(primaryCols) == 1 {
@@ -517,8 +527,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	}
 	orderQuery := query.(*jsonutils.JSONDict).Copy()
 	for _, orderByField := range orderBy {
-		if pagingConf != nil && orderByField == pagingConf.MarkerField {
-			// skip markerField
+		if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
+			// skip markerField in pagingConf
 			continue
 		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
@@ -542,8 +552,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		}
 	}
 	for _, orderByField := range orderBy {
-		if pagingConf != nil && pagingConf.MarkerField == orderByField {
-			// skip markerField
+		if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
+			// skip markerField in pagingConf
 			continue
 		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
@@ -555,39 +565,38 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 			}
 		}
 	}
-	if pagingConf != nil {
-		if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
-			q = q.Asc(pagingConf.MarkerField)
-		} else {
-			q = q.Desc(pagingConf.MarkerField)
-		}
-	}
 
 	if pagingConf != nil {
 		q = q.Limit(int(limit) + 1)
 		if len(pagingMarker) > 0 {
-			if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
-				q = q.GE(pagingConf.MarkerField, pagingMarker)
-			} else {
-				q = q.LE(pagingConf.MarkerField, pagingMarker)
+			markers := decodePagingMarker(pagingMarker)
+			for markerIdx, marker := range markers {
+				if markerIdx < len(pagingConf.MarkerFields) {
+					if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+						q = q.GE(pagingConf.MarkerFields[markerIdx], marker)
+					} else {
+						q = q.LE(pagingConf.MarkerFields[markerIdx], marker)
+					}
+				}
 			}
 		}
 		retList, err := Query2List(manager, ctx, userCred, q, queryDict, false)
 		if err != nil {
 			return nil, httperrors.NewGeneralError(err)
 		}
-		nextMarker := ""
+		nextMarkers := make([]string, 0)
 		if int64(len(retList)) > limit {
-			markerObj, _ := retList[limit].Get(pagingConf.MarkerField)
-			if markerObj != nil {
-				nextMarker = fmt.Sprintf("%s", markerObj)
+			for _, markerField := range pagingConf.MarkerFields {
+				nextMarker, _ := retList[limit].GetString(markerField)
+				nextMarkers = append(nextMarkers, nextMarker)
 			}
 			retList = retList[:limit]
 		}
+		nextMarker := encodePagingMarker(nextMarkers)
 		retResult := modulebase.ListResult{
 			Data: retList, Limit: int(limit),
 			NextMarker:  nextMarker,
-			MarkerField: pagingConf.MarkerField,
+			MarkerField: strings.Join(pagingConf.MarkerFields, ","),
 			MarkerOrder: string(pagingConf.Order),
 		}
 		return &retResult, nil
@@ -1104,9 +1113,17 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		return nil, httperrors.NewForbiddenError("Not allow to create item")
 	}
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+	}
+
 	model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId)
 	if err != nil {
-		log.Errorf("fail to doCreateItem %s", err)
+		// log.Errorf("fail to doCreateItem %s", err)
+		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
+		if failErr != nil {
+			log.Errorf("manager.OnCreateFailed %s", failErr)
+		}
 		return nil, httperrors.NewGeneralError(err)
 	}
 
@@ -1179,54 +1196,71 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 	}
 
 	var (
-		multiData         []jsonutils.JSONObject
-		onBatchCreateFail func()
-		validateError     error
+		multiData []jsonutils.JSONObject
+		// onBatchCreateFail func()
+		// validateError error
 	)
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+	}
+
 	createResults, err := func() ([]sCreateResult, error) {
 		lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
 		defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
 
-		multiData, err = expandMultiCreateParams(data, count)
+		// invoke only Once
+		err = manager.BatchPreValidate(ctx, userCred, ownerId, query, data.(*jsonutils.JSONDict), count)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "manager.BatchPreValidate")
 		}
 
-		ret := make([]sCreateResult, len(multiData))
-		for i, cdata := range multiData {
-			if i == 0 {
-				onBatchCreateFail, validateError = manager.BatchPreValidate(
-					ctx, userCred, ownerId, query, cdata.(*jsonutils.JSONDict), len(multiData))
-				if validateError != nil {
-					return nil, validateError
-				}
-			}
-			model, err := batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, cdata, i)
-			if err != nil && onBatchCreateFail != nil {
-				onBatchCreateFail()
-			}
-			ret[i] = sCreateResult{model: model, err: err}
+		multiData, err = expandMultiCreateParams(data, count)
+		if err != nil {
+			return nil, errors.Wrap(err, "expandMultiCreateParams")
+		}
+
+		// one fail, then all fail
+		ret := make([]sCreateResult, len(multiData))
+		for i := range multiData {
+			var model IModel
+			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i)
+			if err == nil {
+				ret[i] = sCreateResult{model: model, err: nil}
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			for i := range ret {
+				if ret[i].model != nil {
+					DeleteModel(ctx, userCred, ret[i].model)
+					ret[i].model = nil
+				}
+				ret[i].err = err
+			}
+			return nil, errors.Wrap(err, "batchCreateDoCreateItem")
+		} else {
+			return ret, nil
 		}
-		return ret, nil
 	}()
 
 	if err != nil {
-		return nil, err
+		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
+		if failErr != nil {
+			log.Errorf("manager.OnCreateFailed %s", failErr)
+		}
+		return nil, errors.Wrap(err, "createResults")
 	}
+
 	results := make([]modulebase.SubmitResult, count)
 	models := make([]IModel, 0)
 	for i, res := range createResults {
 		result := modulebase.SubmitResult{}
 		if res.err != nil {
-			jsonErr, ok := res.err.(*httputils.JSONClientError)
-			if ok {
-				result.Status = jsonErr.Code
-				result.Data = jsonutils.Marshal(jsonErr)
-			} else {
-				result.Status = 500
-				result.Data = jsonutils.NewString(res.err.Error())
-			}
+			jsonErr := httperrors.NewGeneralError(res.err)
+			result.Status = jsonErr.Code
+			result.Data = jsonutils.Marshal(jsonErr)
 		} else {
 			lockman.LockObject(ctx, res.model)
 			defer lockman.ReleaseObject(ctx, res.model)
@@ -1263,11 +1297,7 @@ func managerPerformCheckCreateData(
 	query jsonutils.JSONObject,
 	data jsonutils.JSONObject,
 ) (jsonutils.JSONObject, error) {
-	body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword())
-	if err != nil {
-		return nil, httperrors.NewGeneralError(err)
-	}
-	bodyDict := body.(*jsonutils.JSONDict)
+	bodyDict := data.(*jsonutils.JSONDict)
 
 	if consts.IsRbacEnabled() {
 		err := isClassRbacAllowed(manager, userCred, ownerId, policy.PolicyActionPerform, action)
@@ -1549,7 +1579,7 @@ func objectUpdateSpec(dispatcher *DBModelDispatcher, model IModel, modelValue re
 
 func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
 	// log.Debugf("Ready to delete %s %s %#v", jsonutils.Marshal(item), item, manager)
-	cleanModelUsages(ctx, userCred, item)
+	// cleanModelUsages(ctx, userCred, item)
 	_, err := Update(item, func() error {
 		return item.MarkDelete()
 	})

pkg/cloudcommon/db/interface.go

@@ -30,11 +30,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
-type IUsage interface {
-	FetchUsage(ctx context.Context) error
-	IsEmpty() bool
-}
-
 type IModelManager interface {
 	lockman.ILockedClass
 	object.IObject
@@ -91,7 +86,9 @@ type IModelManager interface {
 	// ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject)
 	BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
-		query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) (func(), error)
+		query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) error
+
+	OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 
 	// allow perform action
 	AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool

pkg/cloudcommon/db/metadata.go

@@ -23,7 +23,6 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/sqlchemy"
 
@@ -204,7 +203,8 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 		sq := resourceView.Query(field)
 		ownerId, queryScope, err := FetchCheckQueryOwnerScope(ctx, userCred, query, man, policy.PolicyActionList, true)
 		if err != nil {
-			return nil, httperrors.NewGeneralError(errors.Wrap(err, "FetchCheckQueryOwnerScope"))
+			log.Warningf("FetchCheckQueryOwnerScope.%s error: %v", man.Keyword(), err)
+			continue
 		}
 		sq = man.FilterByOwner(sq, ownerId, queryScope)
 		sq = man.FilterBySystemAttributes(sq, userCred, query, queryScope)
@@ -264,7 +264,7 @@ func (manager *SMetadataManager) GetJsonValue(model IModel, key string, userCred
 type sMetadataChange struct {
 	Key    string
 	OValue string
-	NValue string
+	NValue string `json:",allowempty"`
 }
 
 func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, userCred mcclient.TokenCredential) error {

pkg/cloudcommon/db/modelbase.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -381,14 +382,24 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context,
 func (manager *SModelBaseManager) BatchPreValidate(
 	ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
 	query jsonutils.JSONObject, data *jsonutils.JSONDict, count int,
-) (func(), error) {
-	return nil, nil
+) error {
+	return nil
 }
 
 func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	return nil, nil
 }
 
+func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	if CancelPendingUsagesInContext != nil {
+		err := CancelPendingUsagesInContext(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "CancelPendingUsagesInContext")
+		}
+	}
+	return nil
+}
+
 func (model *SModelBase) GetId() string {
 	return ""
 }

pkg/cloudcommon/db/namevalidator.go

@@ -46,7 +46,7 @@ func NewNameValidator(manager IModelManager, ownerId mcclient.IIdentityProvider,
 		return err
 	}
 	if !uniq {
-		return httperrors.NewDuplicateNameError("name", name)
+		return httperrors.NewDuplicateNameError(manager.Keyword(), name)
 	}
 	return nil
 }

pkg/cloudcommon/db/opslog.go

@@ -206,6 +206,7 @@ const (
 
 	ACT_SYNC_CLOUD_DISK          = "sync_cloud_disk"
 	ACT_SYNC_CLOUD_SERVER        = "sync_cloud_server"
+	ACT_SYNC_CLOUD_SKUS          = "sync_cloud_skus"
 	ACT_SYNC_CLOUD_EIP           = "sync_cloud_eip"
 	ACT_SYNC_CLOUD_PROJECT       = "sync_cloud_project"
 	ACT_SYNC_CLOUD_ELASTIC_CACHE = "sync_cloud_elastic_cache"
@@ -275,7 +276,7 @@ type SOpsLog struct {
 	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
 	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`               //= Column(VARCHAR(128, charset='utf8'), nullable=False)
 	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`                //= Column(VARCHAR(32, charset='ascii'), nullable=False)
-	Notes   string `width:"2048" charset:"utf8" list:"user" create:"required"`                               // = Column(VARCHAR(2048, charset='utf8'))
+	Notes   string `charset:"utf8" list:"user" create:"required"`
 
 	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"required" index:"true"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
 	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"required"`                  // tenant    = Column(VARCHAR(128, charset='utf8'))
@@ -630,7 +631,7 @@ func (manager *SOpsLogManager) ResourceScope() rbacutils.TRbacScope {
 func (manager *SOpsLogManager) GetPagingConfig() *SPagingConfig {
 	return &SPagingConfig{
 		Order:        sqlchemy.SQL_ORDER_DESC,
-		MarkerField:  "id",
+		MarkerFields: []string{"id"},
 		DefaultLimit: 20,
 	}
 }

pkg/cloudcommon/db/paging.go

@@ -15,11 +15,38 @@
 package db
 
 import (
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/sqlchemy"
 )
 
 type SPagingConfig struct {
 	Order        sqlchemy.QueryOrderType
-	MarkerField  string
+	MarkerFields []string
 	DefaultLimit int
 }
+
+func decodePagingMarker(markString string) []string {
+	if len(markString) > 2 && markString[0] == '[' && markString[len(markString)-1] == ']' {
+		markJson, _ := jsonutils.ParseString(markString)
+		if markJson != nil {
+			if markArray, ok := markJson.(*jsonutils.JSONArray); ok {
+				return markArray.GetStringArray()
+			}
+		}
+	}
+	if len(markString) > 0 {
+		return []string{markString}
+	}
+	return []string{}
+}
+
+func encodePagingMarker(markers []string) string {
+	switch len(markers) {
+	case 0:
+		return ""
+	case 1:
+		return markers[0]
+	default:
+		return jsonutils.Marshal(markers).String()
+	}
+}

pkg/cloudcommon/db/quotas/context.go

@@ -0,0 +1,88 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package quotas
+
+import (
+	"container/list"
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+const (
+	APP_CONTEXT_KEY_PENDINGUSAGES = appctx.AppContextKey("pendingusages")
+)
+
+func initPendingUsagesInContext(ctx context.Context) context.Context {
+	return context.WithValue(ctx, APP_CONTEXT_KEY_PENDINGUSAGES, list.New())
+}
+
+func appContextPendingUsages(ctx context.Context) []IQuota {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		ret := make([]IQuota, 0)
+		for e := quotaList.Front(); e != nil; e = e.Next() {
+			ret = append(ret, e.Value.(IQuota))
+		}
+		return ret
+	} else {
+		return nil
+	}
+}
+
+func clearPendingUsagesInContext(ctx context.Context) {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		for quotaList.Len() > 0 {
+			quotaList.Remove(quotaList.Front())
+		}
+	}
+}
+
+func savePendingUsagesInContext(ctx context.Context, quotas ...IQuota) {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		for i := range quotas {
+			quotaList.PushBack(quotas[i])
+		}
+	}
+}
+
+func cancelPendingUsagesInContext(ctx context.Context, userCred mcclient.TokenCredential) error {
+	quotas := appContextPendingUsages(ctx)
+	if quotas == nil {
+		return nil
+	}
+	errs := make([]error, 0)
+	for i := range quotas {
+		// cancel and do not save pending usage
+		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i], false)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "CancelPendingUsage %s", jsonutils.Marshal(quotas[i])))
+		}
+	}
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	clearPendingUsagesInContext(ctx)
+	return nil
+}

pkg/cloudcommon/db/quotas/handler.go

@@ -176,7 +176,8 @@ func (manager *SQuotaBaseManager) getQuotaHandler(ctx context.Context, w http.Re
 
 	keys := OwnerIdQuotaKeys(scope, ownerId)
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -188,7 +189,7 @@ func (manager *SQuotaBaseManager) getQuotaHandler(ctx context.Context, w http.Re
 		quota.FetchSystemQuota()
 		manager.SetQuota(ctx, userCred, quota)
 
-		quotaList, err = manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, refresh)
+		quotaList, err = manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, primary, refresh)
 		if err != nil {
 			httperrors.GeneralServerError(w, err)
 			return
@@ -402,7 +403,7 @@ func (manager *SQuotaBaseManager) setQuotaHandler(ctx context.Context, w http.Re
 		}
 	}
 
-	quotaList, err := manager.listQuotas(ctx, userCred, baseKeys.DomainId, baseKeys.ProjectId, scope == rbacutils.ScopeDomain, true)
+	quotaList, err := manager.listQuotas(ctx, userCred, baseKeys.DomainId, baseKeys.ProjectId, scope == rbacutils.ScopeDomain, false, true)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -428,7 +429,8 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 	}
 
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, "", "", false, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, "", "", false, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -470,7 +472,8 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 
 	domainId := owner.GetProjectDomainId()
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, domainId, "", false, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, domainId, "", false, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -478,7 +481,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 	manager.sendQuotaList(w, quotaList)
 }
 
-func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
+func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
 	q := manager.Query()
 	if len(targetDomainId) > 0 {
 		q = q.Equals("domain_id", targetDomainId)
@@ -487,15 +490,27 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 		} else {
 			if len(targetProjectId) > 0 {
 				q = q.Equals("tenant_id", targetProjectId)
-			} else {
-				q = q.IsNotEmpty("tenant_id")
 			}
+			// otherwise, list all projects quotas for a domain
+			// also include the domain's quota
+			// else {
+			// 	q = q.IsNotEmpty("tenant_id")
+			// }
 		}
 	} else {
 		// domain only
 		q = q.IsNotEmpty("domain_id")
 		q = q.IsNullOrEmpty("tenant_id")
 	}
+	if primaryOnly {
+		// list primary quota for domain or project
+		fields := manager.getQuotaFields()
+		for _, f := range fields {
+			if f != "domain_id" && f != "tenant_id" {
+				q = q.IsNullOrEmpty(f)
+			}
+		}
+	}
 	rows, err := q.Rows()
 	if err != nil {
 		if errors.Cause(err) != sql.ErrNoRows {

pkg/cloudcommon/db/quotas/interface.go

@@ -28,6 +28,8 @@ type IQuotaKeys interface {
 	Fields() []string
 	Values() []string
 	Compare(IQuotaKeys) int
+
+	OwnerId() mcclient.IIdentityProvider
 }
 
 type IQuota interface {
@@ -41,6 +43,7 @@ type IQuota interface {
 	Update(quota IQuota)
 	Add(quota IQuota)
 	Sub(quota IQuota)
+	Allocable(quota IQuota) int
 	ResetNegative()
 	Exceed(request IQuota, quota IQuota) error
 	// IsEmpty() bool
@@ -66,8 +69,9 @@ type IQuotaManager interface {
 	db.IResourceModelManager
 
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
-	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error
+	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)
 }

pkg/cloudcommon/db/quotas/quotas.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -40,6 +41,10 @@ func (manager *SQuotaBaseManager) newQuota() IQuota {
 	return model.(IQuota)
 }
 
+func (manager *SQuotaBaseManager) getQuotaFields() []string {
+	return manager.newQuota().GetKeys().Fields()
+}
+
 func (manager *SQuotaBaseManager) cleanPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error {
 	LockQuotaKeys(ctx, manager, keys)
 	defer ReleaseQuotaKeys(ctx, manager, keys)
@@ -61,14 +66,14 @@ func (manager *SQuotaBaseManager) _cleanPendingUsage(ctx context.Context, userCr
 	return nil
 }
 
-func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	LockQuota(ctx, manager, localUsage)
 	defer ReleaseQuota(ctx, manager, localUsage)
 
-	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
-func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	originKeys := localUsage.GetKeys()
 	// currentKeys := cancelUsage.GetKeys()
 
@@ -82,14 +87,16 @@ func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userC
 	}
 	//
 	if localUsage != nil {
-		localUsage.Sub(cancelUsage)
+		localUsage.Sub(pendingUsage)
 	}
 
-	log.Debugf("cancelUsage: %s localUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage))
+	log.Debugf("cancelUsage: %s localUsage: %s pendingUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage), jsonutils.Marshal(pendingUsage))
 
-	err = manager.changeUsage(ctx, userCred, cancelUsage, true)
-	if err != nil {
-		return errors.Wrap(err, "manager.changelUsage")
+	if save {
+		err = manager.changeUsage(ctx, userCred, pendingUsage, true)
+		if err != nil {
+			return errors.Wrap(err, "manager.changelUsage")
+		}
 	}
 
 	return nil
@@ -216,13 +223,21 @@ func (manager *SQuotaBaseManager) checkQuota(ctx context.Context, request IQuota
 
 func (manager *SQuotaBaseManager) __checkQuota(ctx context.Context, quota IQuota, request IQuota) error {
 	keys := quota.GetKeys()
-	log.Debugf("__checkQuota for keys: %s", QuotaKeyString(keys))
+
+	if !consts.GetNonDefaultDomainProjects() {
+		ownerId := keys.OwnerId()
+		if len(ownerId.GetProjectDomainId()) > 0 && len(ownerId.GetProjectId()) == 0 {
+			// if non_default_domain_projects == false
+			// skip domain quota check
+			return nil
+		}
+	}
+
 	used := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, used)
 	if err != nil {
 		return errors.Wrap(err, "manager.usageStore.GetQuotaByKeys")
 	}
-	log.Debugf("__checkQuota usage: %s", jsonutils.Marshal(used))
 	pendings, err := manager.pendingStore.GetChildrenQuotas(ctx, keys)
 	if err != nil {
 		return errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas")
@@ -231,7 +246,6 @@ func (manager *SQuotaBaseManager) __checkQuota(ctx context.Context, quota IQuota
 		if pendings[i].IsEmpty() {
 			continue
 		}
-		log.Debugf("__checkQuota pending %d: %s", i, jsonutils.Marshal(pendings[i]))
 		used.Add(pendings[i])
 	}
 	return used.Exceed(request, quota)
@@ -271,3 +285,51 @@ func (manager *SQuotaBaseManager) _checkSetPendingQuota(ctx context.Context, use
 	}
 	return nil
 }
+
+func (manager *SQuotaBaseManager) getQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
+	quotas, err := manager.GetParentQuotas(ctx, request.GetKeys())
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.getMatchedQuotas")
+	}
+	minCnt := -1
+	for i := len(quotas) - 1; i >= 0; i -= 1 {
+		rel := relation(quotas[i].GetKeys(), pendingKeys)
+		if rel == QuotaKeysContain || rel == QuotaKeysEqual {
+			break
+		}
+		cnt, err := manager.__getQuotaCount(ctx, quotas[i], request)
+		if err != nil {
+			return 0, errors.Wrapf(err, "manager.__getQuotaCount for key %s", QuotaKeyString(quotas[i].GetKeys()))
+		}
+		if minCnt < 0 || minCnt > cnt {
+			minCnt = cnt
+		}
+	}
+	return minCnt, nil
+}
+
+func (manager *SQuotaBaseManager) __getQuotaCount(ctx context.Context, quota IQuota, request IQuota) (int, error) {
+	keys := quota.GetKeys()
+	used := manager.newQuota()
+	err := manager.usageStore.GetQuota(ctx, keys, used)
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.usageStore.GetQuotaByKeys")
+	}
+	pendings, err := manager.pendingStore.GetChildrenQuotas(ctx, keys)
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas")
+	}
+	for i := range pendings {
+		if pendings[i].IsEmpty() {
+			continue
+		}
+		used.Add(pendings[i])
+	}
+	err = used.Exceed(request, quota)
+	if err != nil {
+		return 0, nil
+	}
+	quota.Sub(used)
+	cnt := quota.Allocable(request)
+	return cnt, nil
+}

pkg/cloudcommon/db/quotas/register.go

@@ -20,6 +20,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -33,6 +34,8 @@ func init() {
 	quotaManagerTable = make(map[reflect.Type]IQuotaManager)
 
 	db.CancelUsages = CancelUsages
+	db.CancelPendingUsagesInContext = cancelPendingUsagesInContext
+	db.InitPendingUsagesInContext = initPendingUsagesInContext
 }
 
 func Register(manager IQuotaManager) {
@@ -52,14 +55,19 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 	}
 }
 
-func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	manager := getQuotaManager(cancelUsage)
-	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
 	manager := getQuotaManager(quota)
-	return manager.checkSetPendingQuota(ctx, userCred, quota)
+	err := manager.checkSetPendingQuota(ctx, userCred, quota)
+	if err != nil {
+		return errors.Wrap(err, "manager.checkSetPendingQuota")
+	}
+	savePendingUsagesInContext(ctx, quota)
+	return nil
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
@@ -75,3 +83,8 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
 	}
 }
+
+func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
+	manager := getQuotaManager(request)
+	return manager.getQuotaCount(ctx, request, pendingKeys)
+}

pkg/cloudcommon/db/scoperesource.go

@@ -43,13 +43,23 @@ func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred
 	}
 	switch scope {
 	case rbacutils.ScopeDomain:
-		q = q.Filter(sqlchemy.Equals(q.Field("domain_id"), userCred.GetDomainId()))
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.Equals(q.Field("domain_id"), userCred.GetProjectDomainId()),
+			sqlchemy.IsNullOrEmpty(q.Field("domain_id")),
+		))
 	case rbacutils.ScopeProject:
-		q = q.Filter(sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()))
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()),
+			sqlchemy.IsNullOrEmpty(q.Field("tenant_id")),
+		))
 	}
 	return q
 }
 
+func (m *SScopedResourceBaseManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	return FetchProjectInfo(ctx, data)
+}
+
 func (m *SScopedResourceBaseManager) ValidateCreateData(man IScopedResourceManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	scope, _ := data.GetString("scope")
 	if scope == "" {

pkg/cloudcommon/db/tenantcache.go

@@ -174,8 +174,13 @@ func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context,
 		log.Debugf("fetch empty tenant!!!!\n%s", debug.Stack())
 		return nil, fmt.Errorf("Empty idStr")
 	}
+
+	// It is to query all domain's project.
+	query := jsonutils.NewDict()
+	query.Set("scope", jsonutils.NewString("system"))
+
 	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
-	tenant, err := modules.Projects.GetById(s, idStr, nil)
+	tenant, err := modules.Projects.GetById(s, idStr, query)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
 			return nil, sql.ErrNoRows

pkg/cloudcommon/db/usages.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type IUsage interface {
+	FetchUsage(ctx context.Context) error
+	IsEmpty() bool
+}
+
+var (
+	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
+
+	CancelPendingUsagesInContext func(ctx context.Context, userCred mcclient.TokenCredential) error
+
+	InitPendingUsagesInContext func(ctx context.Context) context.Context
+)

pkg/cloudcommon/db/usercache.go

@@ -19,7 +19,9 @@ import (
 	"database/sql"
 	"fmt"
 	"runtime/debug"
+	"time"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
@@ -111,14 +113,25 @@ func (manager *SUserCacheManager) FetchUserFromKeystone(ctx context.Context, idS
 		log.Debugf("fetch empty user!!!!\n%s", debug.Stack())
 		return nil, fmt.Errorf("Empty idStr")
 	}
+
+	// It's to query the full list of users(contains other domain's ones and system ones)
+	query := jsonutils.NewDict()
+	query.Set("scope", jsonutils.NewString("system"))
+	query.Set("system", jsonutils.JSONTrue)
+
 	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
-	user, err := modules.UsersV3.GetById(s, idStr, nil)
+	user, err := modules.UsersV3.GetById(s, idStr, query)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
-			return nil, sql.ErrNoRows
+			user, err = modules.UsersV3.GetByName(s, idStr, query)
+			if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
+				return nil, sql.ErrNoRows
+			}
+		}
+		if err != nil {
+			log.Errorf("fetch user %s fail %s", idStr, err)
+			return nil, errors.Wrap(err, "modules.UsersV3.Get")
 		}
-		log.Errorf("fetch user %s fail %s", idStr, err)
-		return nil, errors.Wrap(err, "modules.UsersV3.Get")
 	}
 	id, _ := user.GetString("id")
 	name, _ := user.GetString("name")
@@ -133,7 +146,7 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s
 
 	objo, err := manager.FetchById(idStr)
 	if err != nil && err != sql.ErrNoRows {
-		log.Errorf("FetchTenantbyId fail %s", err)
+		log.Errorf("FetchUserbyId fail %s", err)
 		return nil, err
 	}
 	if err == nil {
@@ -143,6 +156,7 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s
 			obj.Name = name
 			obj.Domain = domain
 			obj.DomainId = domainId
+			obj.LastCheck = time.Now().UTC()
 			return nil
 		})
 		if err != nil {
@@ -157,7 +171,8 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s
 		obj.Name = name
 		obj.Domain = domain
 		obj.DomainId = domainId
-		err = manager.TableSpec().Insert(obj)
+		obj.LastCheck = time.Now().UTC()
+		err = manager.TableSpec().InsertOrUpdate(obj)
 		if err != nil {
 			return nil, err
 		} else {

pkg/cloudcommon/options/changes.go

@@ -14,25 +14,39 @@
 
 package options
 
+import (
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+)
+
 func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 	oldOpts := oOpts.(*BaseOptions)
 	newOpts := nOpts.(*BaseOptions)
 
+	changed := false
 	if oldOpts.RequestWorkerCount != newOpts.RequestWorkerCount {
-		return true
+		changed = true
 	}
 	if oldOpts.TimeZone != newOpts.TimeZone {
-		return true
+		changed = true
 	}
-	return false
+	if oldOpts.EnableRbac != newOpts.EnableRbac {
+		changed = true
+	}
+	if oldOpts.NonDefaultDomainProjects != newOpts.NonDefaultDomainProjects {
+		consts.SetNonDefaultDomainProjects(newOpts.NonDefaultDomainProjects)
+		changed = true
+	}
+	return changed
 }
 
 func OnCommonOptionsChange(oOpts, nOpts interface{}) bool {
 	oldOpts := oOpts.(*CommonOptions)
 	newOpts := nOpts.(*CommonOptions)
 
+	changed := false
 	if OnBaseOptionsChange(&oldOpts.BaseOptions, &newOpts.BaseOptions) {
-		return true
+		changed = true
 	}
-	return false
+
+	return changed
 }

pkg/cloudcommon/options/manager.go

@@ -85,7 +85,7 @@ func (manager *SOptionManager) doSync(first bool) {
 
 	if merged && !reflect.DeepEqual(newOpts, manager.options) {
 		log.Infof("Service config changed ...")
-		if !first && manager.onOptionsChange != nil && manager.onOptionsChange(manager.options, newOpts) {
+		if manager.onOptionsChange != nil && manager.onOptionsChange(manager.options, newOpts) && !first {
 			log.Infof("Option changes detected and going to restart the program...")
 			appsrv.SetExitFlag()
 		}

pkg/cloudcommon/options/mergeconf.go

@@ -82,9 +82,12 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 		serviceConf, err := getServiceConfig(s.session, s.serviceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", serviceType, err)
-		} else {
+		} else if serviceConf != nil {
 			s.config.Update(serviceConf)
 			merged = true
+		} else {
+			// not initialized
+			s.Upload()
 		}
 	}
 	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
@@ -92,9 +95,11 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 		commonConf, err := getServiceConfig(s.session, commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
-		} else {
+		} else if commonConf != nil {
 			s.config.Update(commonConf)
 			merged = true
+		} else {
+			// common not initialized
 		}
 	}
 	if merged {

pkg/cloudcommon/options/options.go

@@ -58,7 +58,7 @@ type BaseOptions struct {
 	TempPath  string   `help:"Path for store temp file, at least 40G space" default:"/opt/yunion/tmp"`
 
 	ApplicationID      string `help:"Application ID"`
-	RequestWorkerCount int    `default:"4" help:"Request worker thread count, default is 4"`
+	RequestWorkerCount int    `default:"8" help:"Request worker thread count, default is 8"`
 
 	EnableSsl   bool   `help:"Enable https"`
 	SslCaCerts  string `help:"ssl certificate ca root file, separating ca and cert file is not encouraged" alias:"ca-file"`