Merge pull request #4796 from tb365/automated-cherry-pick-of-#4795-upstream-release-3.0

Automated cherry pick of #4795: add filter skus by domain id

build/docker/Dockerfile.ansibleserver

@@ -1,15 +1,3 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
-
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk update && \
-    apk add --no-cache tzdata ansible ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+FROM registry.cn-beijing.aliyuncs.com/yunionio/ansibleserver-base:latest
 
 ADD ./_output/bin/ansibleserver /opt/yunion/bin/ansibleserver

build/docker/Dockerfile.ansibleserver-base

@@ -0,0 +1,13 @@
+FROM frolvlad/alpine-glibc:glibc-2.28
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk update && \
+    apk add --no-cache tzdata ansible ca-certificates && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/docker/Dockerfile.baremetal-base

@@ -27,3 +27,8 @@ COPY ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal
 COPY ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal
+
+# add mkisofs with bundles libraries
+ADD ./mkisofs /bin/mkisofs
+ADD ./.mkisofs.bin /bin/.mkisofs.bin
+ADD ./bundles/mkisofs /bin/bundles/mkisofs

build/docker/Dockerfile.climc

@@ -1,16 +1,3 @@
-FROM frolvlad/alpine-glibc:glibc-2.28
-
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN apk add --no-cache bash bash-completion tzdata ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
-
-ENV PATH="/opt/yunion/bin:${PATH}"
-
-RUN mkdir -p /opt/yunion/bin
+FROM registry.cn-beijing.aliyuncs.com/yunionio/climc-base:latest
 
 ADD ./_output/bin/climc ./_output/bin/*cli /opt/yunion/bin/

build/docker/Dockerfile.climc-base

@@ -0,0 +1,16 @@
+FROM frolvlad/alpine-glibc:glibc-2.28
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk add --no-cache bash bash-completion tzdata ca-certificates && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+ENV PATH="/opt/yunion/bin:${PATH}"
+
+RUN mkdir -p /opt/yunion/bin

build/docker/Dockerfile.esxi-agent

@@ -0,0 +1,4 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+
+COPY ./build/esxi-agent/root/opt/ /opt/
+ADD ./_output/bin/esxi-agent /opt/yunion/bin/esxi-agent

build/docker/Dockerfile.webconsole

@@ -1,14 +1,3 @@
-FROM yunion/kubectl:1.14.3
-
-MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
-
-ENV TZ Asia/Shanghai
-
-RUN mkdir -p /opt/yunion/bin
-
-RUN apk add --no-cache sshpass ipmitool openssh-client tzdata ca-certificates && \
-    rm -rf /var/cache/apk/*
-
-RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+FROM registry.cn-beijing.aliyuncs.com/yunionio/webconsole-base:latest
 
 ADD ./_output/bin/webconsole /opt/yunion/bin/webconsole

build/docker/Dockerfile.webconsole-base

@@ -0,0 +1,12 @@
+FROM yunion/kubectl:1.14.3
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk add --no-cache sshpass ipmitool openssh-client tzdata ca-certificates && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

build/esxi-agent/root/opt/yunion/share/vmware/ovf.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Envelope xmlns="http://schemas.dmtf.org/ovf/envelope/1"
+          xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1"
+          xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common"
+          xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData"
+          xmlns:vmw="http://www.vmware.com/schema/ovf"
+          xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <References>
+    <File ovf:href="{{ .Name }}" ovf:id="file1" ovf:size="{{ .Size }}"/>
+  </References>
+  <DiskSection>
+    <Info>Virtual disk information</Info>
+    <Disk ovf:capacity="{{ .Capacity }}" ovf:capacityAllocationUnits="byte" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized" ovf:populatedSize="0"/>
+  </DiskSection>
+  <VirtualSystem ovf:id="{{ .ImportName }}">
+    <Info>A virtual machine</Info>
+    <Name>{{ .ImportName }}</Name>
+    <OperatingSystemSection ovf:id="100" vmw:osType="other26xLinux64Guest">
+      <Info>The kind of installed guest operating system</Info>
+    </OperatingSystemSection>
+    <VirtualHardwareSection>
+      <Info>Virtual hardware requirements</Info>
+      <System>
+        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
+        <vssd:InstanceID>0</vssd:InstanceID>
+        <vssd:VirtualSystemIdentifier>{{ .ImportName }}</vssd:VirtualSystemIdentifier>
+        <vssd:VirtualSystemType>vmx-07</vssd:VirtualSystemType>
+      </System>
+      <Item>
+        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
+        <rasd:Description>Number of Virtual CPUs</rasd:Description>
+        <rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
+        <rasd:InstanceID>1</rasd:InstanceID>
+        <rasd:ResourceType>3</rasd:ResourceType>
+        <rasd:VirtualQuantity>1</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
+        <rasd:Description>Memory Size</rasd:Description>
+        <rasd:ElementName>1024MB of memory</rasd:ElementName>
+        <rasd:InstanceID>2</rasd:InstanceID>
+        <rasd:ResourceType>4</rasd:ResourceType>
+        <rasd:VirtualQuantity>1024</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:Address>0</rasd:Address>
+        <rasd:Description>SCSI Controller</rasd:Description>
+        <rasd:ElementName>SCSI Controller 0</rasd:ElementName>
+        <rasd:InstanceID>3</rasd:InstanceID>
+        <rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
+        <rasd:ResourceType>6</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:ElementName>Hard Disk 1</rasd:ElementName>
+        <rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
+        <rasd:InstanceID>9</rasd:InstanceID>
+        <rasd:Parent>3</rasd:Parent>
+        <rasd:ResourceType>17</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false"/>
+      </Item>
+    </VirtualHardwareSection>
+  </VirtualSystem>
+</Envelope>

build/esxi-agent/vars

@@ -0,0 +1,2 @@
+DESCRIPTION="Esxi Agent"
+# SERVICE="yes"

cmd/climc/shell/cloudaccounts.go

@@ -88,6 +88,16 @@ func init() {
 	R(&options.SAliyunCloudAccountCreateOptions{}, "cloud-account-create-aliyun", "Create an Aliyun cloud account", func(s *mcclient.ClientSession, args *options.SAliyunCloudAccountCreateOptions) error {
 		params := jsonutils.Marshal(args)
 		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Aliyun"), "provider")
+		options := jsonutils.NewDict()
+		if len(args.OptionsBillingReportBucket) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
+		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if options.Size() > 0 {
+			params.(*jsonutils.JSONDict).Add(options, "options")
+		}
 		result, err := modules.Cloudaccounts.Create(s, params)
 		if err != nil {
 			return err
@@ -147,6 +157,12 @@ func init() {
 		if len(args.OptionsBillingReportBucket) > 0 {
 			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
 		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFileAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFileAccount), "billing_file_account")
+		}
 		if options.Size() > 0 {
 			params.Add(options, "options")
 		}
@@ -280,6 +296,28 @@ func init() {
 		if params.Size() == 0 {
 			return InvalidUpdateError()
 		}
+
+		options := jsonutils.NewDict()
+		if len(args.OptionsBillingReportBucket) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
+		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if options.Size() > 0 {
+			params.Add(options, "options")
+		}
+		removeOptions := make([]string, 0)
+		if args.RemoveOptionsBillingReportBucket {
+			removeOptions = append(removeOptions, "billing_report_bucket")
+		}
+		if args.RemoveOptionsBillingBucketAccount {
+			removeOptions = append(removeOptions, "billing_bucket_account")
+		}
+		if len(removeOptions) > 0 {
+			params.Add(jsonutils.NewStringArray(removeOptions), "remove_options")
+		}
+
 		result, err := modules.Cloudaccounts.Update(s, args.ID, params)
 		if err != nil {
 			return err
@@ -337,6 +375,12 @@ func init() {
 		if len(args.OptionsBillingReportBucket) > 0 {
 			options.Add(jsonutils.NewString(args.OptionsBillingReportBucket), "billing_report_bucket")
 		}
+		if len(args.OptionsBillingBucketAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingBucketAccount), "billing_bucket_account")
+		}
+		if len(args.OptionsBillingFileAccount) > 0 {
+			options.Add(jsonutils.NewString(args.OptionsBillingFileAccount), "billing_file_account")
+		}
 		if options.Size() > 0 {
 			params.Add(options, "options")
 		}
@@ -344,6 +388,12 @@ func init() {
 		if args.RemoveOptionsBillingReportBucket {
 			removeOptions = append(removeOptions, "billing_report_bucket")
 		}
+		if args.RemoveOptionsBillingBucketAccount {
+			removeOptions = append(removeOptions, "billing_bucket_account")
+		}
+		if args.RemoveOptionsBillingFileAccount {
+			removeOptions = append(removeOptions, "billing_file_account")
+		}
 		if len(removeOptions) > 0 {
 			params.Add(jsonutils.NewStringArray(removeOptions), "remove_options")
 		}
@@ -727,4 +777,34 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type CloudaccountSyncSkusOptions struct {
+		ID       string `help:"ID or Name of cloud account"`
+		RESOURCE string `help:"Resource of skus" choices:"serversku|elasticcachesku|dbinstance_sku"`
+		Force    bool   `help:"Force sync no matter what"`
+		Provider string `help:"provider to sync"`
+		Region   string `help:"region to sync"`
+	}
+	R(&CloudaccountSyncSkusOptions{}, "cloud-account-sync-skus", "Sync skus of a cloud account", func(s *mcclient.ClientSession, args *CloudaccountSyncSkusOptions) error {
+		params := jsonutils.NewDict()
+		params.Set("resource", jsonutils.NewString(args.RESOURCE))
+		if args.Force {
+			params.Add(jsonutils.JSONTrue, "force")
+		}
+
+		if len(args.Provider) > 0 {
+			params.Add(jsonutils.NewString(args.Provider), "cloudprovider")
+		}
+
+		if len(args.Region) > 0 {
+			params.Add(jsonutils.NewString(args.Region), "cloudregion")
+		}
+
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "sync-skus", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/quotas.go

@@ -24,7 +24,7 @@ import (
 )
 
 type ComputeQuotaKeys struct {
-	RegionQuotaKeys
+	ZoneQuotaKeys
 
 	Hypervisor string `help:"hypervisor" choices:"kvm|baremetal"`
 }
@@ -53,8 +53,8 @@ type RegionQuotaOptions struct {
 
 	Port  int64 `help:"Internal NIC count" json:"port,omitzero"`
 	Eport int64 `help:"External NIC count" json:"eport,omitzero"`
-	Bw    int64 `help:"Internal bandwidth in Mbps" json:"bw,omitzero"`
-	Ebw   int64 `help:"External bandwidth in Mbps" json:"ebw,omitzero"`
+	// Bw    int64 `help:"Internal bandwidth in Mbps" json:"bw,omitzero"`
+	// Ebw   int64 `help:"External bandwidth in Mbps" json:"ebw,omitzero"`
 
 	Eip int64 `help:"Elastic IP count" json:"eip,omitzero"`
 
@@ -110,6 +110,7 @@ func init() {
 		Project string `help:"Tenant name or ID" json:"tenant"`
 		Domain  string `help:"Domain name or ID" json:"domain"`
 		Refresh bool   `help:"refresh" json:"refresh,omitfalse"`
+		Primary bool   `help:"get primary quota ONLY" json:"primary,omitfalse"`
 	}
 	R(&QuotaOptions{}, "quota", "Show quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
 		params := jsonutils.Marshal(args)
@@ -239,6 +240,8 @@ func init() {
 
 	type QuotaListOptions struct {
 		ProjectDomain string `help:"domain name or ID to query project quotas"`
+		Refresh       bool   `help:"refresh" json:"refresh,omitfalse"`
+		Primary       bool   `help:"list primary quota ONLY" json:"primary,omitfalse"`
 	}
 	R(&QuotaListOptions{}, "quota-list", "List quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
 		params := jsonutils.Marshal(args)

cmd/esxi-agent/main.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/esxi/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
+
+func main() {
+	defer atexit.Handle()
+
+	service.New().StartService()
+}

pkg/apigateway/handler/misc.go

@@ -285,6 +285,12 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW
 		domain := row[2]
 		allowWebConsole := strings.ToLower(row[3])
 
+		// 忽略空白行
+		rowStr := strings.Join(row, "")
+		if len(strings.TrimSpace(rowStr)) == 0 {
+			continue
+		}
+
 		if len(name) == 0 {
 			e := httperrors.NewClientError("row %d name is empty", rowIdx)
 			httperrors.JsonClientError(w, e)

pkg/apis/compute/cloudaccount.go

@@ -27,6 +27,7 @@ type CloudaccountCreateInput struct {
 	apis.EnabledStatusStandaloneResourceCreateInput
 
 	Provider            string
+	AccountId           string
 	Brand               string
 	IsPublicCloud       bool
 	IsOnPremise         bool

pkg/apis/compute/guesttemplate.go

@@ -55,6 +55,7 @@ type GuesttemplateConfigInfo struct {
 	Secgroup             string                 `json:"secgroup"`
 	IsolatedDeviceConfig []IsolatedDeviceConfig `json:"isolated_device_config"`
 	Image                string                 `json:"image"`
+	ResetPassword        bool                   `json:"reset_password"`
 }
 
 type GuesttemplateDisk struct {

pkg/apis/scheduler/api.go

@@ -77,6 +77,8 @@ type ScheduleInput struct {
 	LiveMigrate  bool   `json:"live_migrate"`
 	CpuDesc      string `json:"cpu_desc"`
 	CpuMicrocode string `json:"cpu_microcode"`
+
+	PendingUsages []jsonutils.JSONObject
 }
 
 func (input ScheduleInput) ToConditionInput() *jsonutils.JSONDict {

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -688,7 +688,7 @@ func (adapter *MegaRaidAdaptor) storcliIsJBODEnabled() bool {
 		line = strings.ToLower(line)
 		if strings.HasPrefix(line, "jbod") {
 			data := strings.Split(line, " ")
-			if strings.TrimSpace(data[1]) == "on" {
+			if strings.TrimSpace(data[len(data)-1]) == "on" {
 				return true
 			}
 			return false

pkg/cloudcommon/agent/agent.go

@@ -24,6 +24,8 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/version"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/agent/iagent"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -31,19 +33,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
-type IAgent interface {
-	GetAgentType() string
-	GetAccessIP() (net.IP, error)
-	GetListenIP() (net.IP, error)
-	GetPort() int
-	GetEnableSsl() bool
-	GetZoneName() string
-	GetAdminSession() *mcclient.ClientSession
-	TuneSystem() error
-	StartService() error
-	StopService() error
-}
-
 type SZoneInfo struct {
 	Name string `json:"name"`
 	Id   string `json:"id"`
@@ -80,11 +69,11 @@ func getIfaceIPs(iface *net.Interface) ([]net.IP, error) {
 	return ips, nil
 }
 
-func (agent *SBaseAgent) IAgent() IAgent {
-	return agent.GetVirtualObject().(IAgent)
+func (agent *SBaseAgent) IAgent() iagent.IAgent {
+	return agent.GetVirtualObject().(iagent.IAgent)
 }
 
-func (agent *SBaseAgent) Init(iagent IAgent, ifname string, cachePath string) error {
+func (agent *SBaseAgent) Init(iagent iagent.IAgent, ifname string, cachePath string) error {
 	iface, err := net.InterfaceByName(ifname)
 	if err != nil {
 		return err
@@ -246,7 +235,9 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
 	if err != nil {
 		return err
 	}
-	params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
+	if agent.IAgent().GetAgentType() != string(api.AgentTypeEsxi) {
+		params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
+	}
 	params.Add(jsonutils.NewString(agent.IAgent().GetAgentType()), "agent_type")
 	ret, err := modules.Baremetalagents.List(session, params)
 	if err != nil {

pkg/cloudcommon/agent/iagent/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iagent // import "yunion.io/x/onecloud/pkg/cloudcommon/agent/iagent"

pkg/cloudcommon/agent/iagent/interface.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iagent
+
+import (
+	"net"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type IAgent interface {
+	GetAgentType() string
+	GetAccessIP() (net.IP, error)
+	GetListenIP() (net.IP, error)
+	GetPort() int
+	GetEnableSsl() bool
+	GetZoneName() string
+	GetAdminSession() *mcclient.ClientSession
+	TuneSystem() error
+	StartService() error
+	StopService() error
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/filterclause"
 	"yunion.io/x/pkg/utils"
@@ -38,15 +39,10 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
-	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
-var (
-	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
-)
-
 type DBModelDispatcher struct {
 	modelManager IModelManager
 }
@@ -578,10 +574,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		}
 		nextMarker := ""
 		if int64(len(retList)) > limit {
-			markerObj, _ := retList[limit].Get(pagingConf.MarkerField)
-			if markerObj != nil {
-				nextMarker = fmt.Sprintf("%s", markerObj)
-			}
+			nextMarker, _ = retList[limit].GetString(pagingConf.MarkerField)
 			retList = retList[:limit]
 		}
 		retResult := modulebase.ListResult{
@@ -1104,9 +1097,17 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		return nil, httperrors.NewForbiddenError("Not allow to create item")
 	}
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+	}
+
 	model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId)
 	if err != nil {
-		log.Errorf("fail to doCreateItem %s", err)
+		// log.Errorf("fail to doCreateItem %s", err)
+		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
+		if failErr != nil {
+			log.Errorf("manager.OnCreateFailed %s", failErr)
+		}
 		return nil, httperrors.NewGeneralError(err)
 	}
 
@@ -1179,54 +1180,71 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 	}
 
 	var (
-		multiData         []jsonutils.JSONObject
-		onBatchCreateFail func()
-		validateError     error
+		multiData []jsonutils.JSONObject
+		// onBatchCreateFail func()
+		// validateError error
 	)
 
+	if InitPendingUsagesInContext != nil {
+		ctx = InitPendingUsagesInContext(ctx)
+	}
+
 	createResults, err := func() ([]sCreateResult, error) {
 		lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
 		defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
 
-		multiData, err = expandMultiCreateParams(data, count)
+		// invoke only Once
+		err = manager.BatchPreValidate(ctx, userCred, ownerId, query, data.(*jsonutils.JSONDict), count)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "manager.BatchPreValidate")
 		}
 
-		ret := make([]sCreateResult, len(multiData))
-		for i, cdata := range multiData {
-			if i == 0 {
-				onBatchCreateFail, validateError = manager.BatchPreValidate(
-					ctx, userCred, ownerId, query, cdata.(*jsonutils.JSONDict), len(multiData))
-				if validateError != nil {
-					return nil, validateError
-				}
-			}
-			model, err := batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, cdata, i)
-			if err != nil && onBatchCreateFail != nil {
-				onBatchCreateFail()
-			}
-			ret[i] = sCreateResult{model: model, err: err}
+		multiData, err = expandMultiCreateParams(data, count)
+		if err != nil {
+			return nil, errors.Wrap(err, "expandMultiCreateParams")
+		}
+
+		// one fail, then all fail
+		ret := make([]sCreateResult, len(multiData))
+		for i := range multiData {
+			var model IModel
+			model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i)
+			if err == nil {
+				ret[i] = sCreateResult{model: model, err: nil}
+			} else {
+				break
+			}
+		}
+		if err != nil {
+			for i := range ret {
+				if ret[i].model != nil {
+					DeleteModel(ctx, userCred, ret[i].model)
+					ret[i].model = nil
+				}
+				ret[i].err = err
+			}
+			return nil, errors.Wrap(err, "batchCreateDoCreateItem")
+		} else {
+			return ret, nil
 		}
-		return ret, nil
 	}()
 
 	if err != nil {
-		return nil, err
+		failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
+		if failErr != nil {
+			log.Errorf("manager.OnCreateFailed %s", failErr)
+		}
+		return nil, errors.Wrap(err, "createResults")
 	}
+
 	results := make([]modulebase.SubmitResult, count)
 	models := make([]IModel, 0)
 	for i, res := range createResults {
 		result := modulebase.SubmitResult{}
 		if res.err != nil {
-			jsonErr, ok := res.err.(*httputils.JSONClientError)
-			if ok {
-				result.Status = jsonErr.Code
-				result.Data = jsonutils.Marshal(jsonErr)
-			} else {
-				result.Status = 500
-				result.Data = jsonutils.NewString(res.err.Error())
-			}
+			jsonErr := httperrors.NewGeneralError(res.err)
+			result.Status = jsonErr.Code
+			result.Data = jsonutils.Marshal(jsonErr)
 		} else {
 			lockman.LockObject(ctx, res.model)
 			defer lockman.ReleaseObject(ctx, res.model)

pkg/cloudcommon/db/interface.go

@@ -30,11 +30,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
-type IUsage interface {
-	FetchUsage(ctx context.Context) error
-	IsEmpty() bool
-}
-
 type IModelManager interface {
 	lockman.ILockedClass
 	object.IObject
@@ -91,7 +86,9 @@ type IModelManager interface {
 	// ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
 	OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject)
 	BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
-		query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) (func(), error)
+		query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) error
+
+	OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error
 
 	// allow perform action
 	AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool

pkg/cloudcommon/db/metadata.go

@@ -264,7 +264,7 @@ func (manager *SMetadataManager) GetJsonValue(model IModel, key string, userCred
 type sMetadataChange struct {
 	Key    string
 	OValue string
-	NValue string
+	NValue string `json:",allowempty"`
 }
 
 func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, userCred mcclient.TokenCredential) error {

pkg/cloudcommon/db/modelbase.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -381,14 +382,24 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context,
 func (manager *SModelBaseManager) BatchPreValidate(
 	ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
 	query jsonutils.JSONObject, data *jsonutils.JSONDict, count int,
-) (func(), error) {
-	return nil, nil
+) error {
+	return nil
 }
 
 func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	return nil, nil
 }
 
+func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	if CancelPendingUsagesInContext != nil {
+		err := CancelPendingUsagesInContext(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "CancelPendingUsagesInContext")
+		}
+	}
+	return nil
+}
+
 func (model *SModelBase) GetId() string {
 	return ""
 }

pkg/cloudcommon/db/namevalidator.go

@@ -46,7 +46,7 @@ func NewNameValidator(manager IModelManager, ownerId mcclient.IIdentityProvider,
 		return err
 	}
 	if !uniq {
-		return httperrors.NewDuplicateNameError("name", name)
+		return httperrors.NewDuplicateNameError(manager.Keyword(), name)
 	}
 	return nil
 }

pkg/cloudcommon/db/opslog.go

@@ -206,6 +206,7 @@ const (
 
 	ACT_SYNC_CLOUD_DISK          = "sync_cloud_disk"
 	ACT_SYNC_CLOUD_SERVER        = "sync_cloud_server"
+	ACT_SYNC_CLOUD_SKUS          = "sync_cloud_skus"
 	ACT_SYNC_CLOUD_EIP           = "sync_cloud_eip"
 	ACT_SYNC_CLOUD_PROJECT       = "sync_cloud_project"
 	ACT_SYNC_CLOUD_ELASTIC_CACHE = "sync_cloud_elastic_cache"

pkg/cloudcommon/db/quotas/context.go

@@ -0,0 +1,87 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package quotas
+
+import (
+	"container/list"
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+const (
+	APP_CONTEXT_KEY_PENDINGUSAGES = appctx.AppContextKey("pendingusages")
+)
+
+func initPendingUsagesInContext(ctx context.Context) context.Context {
+	return context.WithValue(ctx, APP_CONTEXT_KEY_PENDINGUSAGES, list.New())
+}
+
+func appContextPendingUsages(ctx context.Context) []IQuota {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		ret := make([]IQuota, 0)
+		for e := quotaList.Front(); e != nil; e = e.Next() {
+			ret = append(ret, e.Value.(IQuota))
+		}
+		return ret
+	} else {
+		return nil
+	}
+}
+
+func clearPendingUsagesInContext(ctx context.Context) {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		for quotaList.Len() > 0 {
+			quotaList.Remove(quotaList.Front())
+		}
+	}
+}
+
+func savePendingUsagesInContext(ctx context.Context, quotas ...IQuota) {
+	val := ctx.Value(APP_CONTEXT_KEY_PENDINGUSAGES)
+	if val != nil {
+		quotaList := val.(*list.List)
+		for i := range quotas {
+			quotaList.PushBack(quotas[i])
+		}
+	}
+}
+
+func cancelPendingUsagesInContext(ctx context.Context, userCred mcclient.TokenCredential) error {
+	quotas := appContextPendingUsages(ctx)
+	if quotas == nil {
+		return nil
+	}
+	errs := make([]error, 0)
+	for i := range quotas {
+		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i])
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "CancelPendingUsage %s", jsonutils.Marshal(quotas[i])))
+		}
+	}
+	if len(errs) > 0 {
+		return errors.NewAggregate(errs)
+	}
+	clearPendingUsagesInContext(ctx)
+	return nil
+}

pkg/cloudcommon/db/quotas/handler.go

@@ -176,7 +176,8 @@ func (manager *SQuotaBaseManager) getQuotaHandler(ctx context.Context, w http.Re
 
 	keys := OwnerIdQuotaKeys(scope, ownerId)
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -188,7 +189,7 @@ func (manager *SQuotaBaseManager) getQuotaHandler(ctx context.Context, w http.Re
 		quota.FetchSystemQuota()
 		manager.SetQuota(ctx, userCred, quota)
 
-		quotaList, err = manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, refresh)
+		quotaList, err = manager.listQuotas(ctx, userCred, keys.DomainId, keys.ProjectId, scope == rbacutils.ScopeDomain, primary, refresh)
 		if err != nil {
 			httperrors.GeneralServerError(w, err)
 			return
@@ -402,7 +403,7 @@ func (manager *SQuotaBaseManager) setQuotaHandler(ctx context.Context, w http.Re
 		}
 	}
 
-	quotaList, err := manager.listQuotas(ctx, userCred, baseKeys.DomainId, baseKeys.ProjectId, scope == rbacutils.ScopeDomain, true)
+	quotaList, err := manager.listQuotas(ctx, userCred, baseKeys.DomainId, baseKeys.ProjectId, scope == rbacutils.ScopeDomain, false, true)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -428,7 +429,8 @@ func (manager *SQuotaBaseManager) listDomainQuotaHandler(ctx context.Context, w
 	}
 
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, "", "", false, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, "", "", false, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -470,7 +472,8 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 
 	domainId := owner.GetProjectDomainId()
 	refresh := jsonutils.QueryBoolean(query, "refresh", false)
-	quotaList, err := manager.listQuotas(ctx, userCred, domainId, "", false, refresh)
+	primary := jsonutils.QueryBoolean(query, "primary", false)
+	quotaList, err := manager.listQuotas(ctx, userCred, domainId, "", false, primary, refresh)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -478,7 +481,7 @@ func (manager *SQuotaBaseManager) listProjectQuotaHandler(ctx context.Context, w
 	manager.sendQuotaList(w, quotaList)
 }
 
-func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
+func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mcclient.TokenCredential, targetDomainId string, targetProjectId string, domainOnly bool, primaryOnly bool, refresh bool) ([]jsonutils.JSONObject, error) {
 	q := manager.Query()
 	if len(targetDomainId) > 0 {
 		q = q.Equals("domain_id", targetDomainId)
@@ -487,15 +490,27 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli
 		} else {
 			if len(targetProjectId) > 0 {
 				q = q.Equals("tenant_id", targetProjectId)
-			} else {
-				q = q.IsNotEmpty("tenant_id")
 			}
+			// otherwise, list all projects quotas for a domain
+			// also include the domain's quota
+			// else {
+			// 	q = q.IsNotEmpty("tenant_id")
+			// }
 		}
 	} else {
 		// domain only
 		q = q.IsNotEmpty("domain_id")
 		q = q.IsNullOrEmpty("tenant_id")
 	}
+	if primaryOnly {
+		// list primary quota for domain or project
+		fields := manager.getQuotaFields()
+		for _, f := range fields {
+			if f != "domain_id" && f != "tenant_id" {
+				q = q.IsNullOrEmpty(f)
+			}
+		}
+	}
 	rows, err := q.Rows()
 	if err != nil {
 		if errors.Cause(err) != sql.ErrNoRows {

pkg/cloudcommon/db/quotas/interface.go

@@ -41,6 +41,7 @@ type IQuota interface {
 	Update(quota IQuota)
 	Add(quota IQuota)
 	Sub(quota IQuota)
+	Allocable(quota IQuota) int
 	ResetNegative()
 	Exceed(request IQuota, quota IQuota) error
 	// IsEmpty() bool
@@ -68,6 +69,7 @@ type IQuotaManager interface {
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
 	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
+	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 
 	FetchIdNames(ctx context.Context, idMap map[string]map[string]string) (map[string]map[string]string, error)
 }

pkg/cloudcommon/db/quotas/quotas.go

@@ -40,6 +40,10 @@ func (manager *SQuotaBaseManager) newQuota() IQuota {
 	return model.(IQuota)
 }
 
+func (manager *SQuotaBaseManager) getQuotaFields() []string {
+	return manager.newQuota().GetKeys().Fields()
+}
+
 func (manager *SQuotaBaseManager) cleanPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, keys IQuotaKeys) error {
 	LockQuotaKeys(ctx, manager, keys)
 	defer ReleaseQuotaKeys(ctx, manager, keys)
@@ -82,12 +86,12 @@ func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userC
 	}
 	//
 	if localUsage != nil {
-		localUsage.Sub(cancelUsage)
+		localUsage.Sub(pendingUsage)
 	}
 
-	log.Debugf("cancelUsage: %s localUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage))
+	log.Debugf("cancelUsage: %s localUsage: %s pendingUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage), jsonutils.Marshal(pendingUsage))
 
-	err = manager.changeUsage(ctx, userCred, cancelUsage, true)
+	err = manager.changeUsage(ctx, userCred, pendingUsage, true)
 	if err != nil {
 		return errors.Wrap(err, "manager.changelUsage")
 	}
@@ -216,13 +220,11 @@ func (manager *SQuotaBaseManager) checkQuota(ctx context.Context, request IQuota
 
 func (manager *SQuotaBaseManager) __checkQuota(ctx context.Context, quota IQuota, request IQuota) error {
 	keys := quota.GetKeys()
-	log.Debugf("__checkQuota for keys: %s", QuotaKeyString(keys))
 	used := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, used)
 	if err != nil {
 		return errors.Wrap(err, "manager.usageStore.GetQuotaByKeys")
 	}
-	log.Debugf("__checkQuota usage: %s", jsonutils.Marshal(used))
 	pendings, err := manager.pendingStore.GetChildrenQuotas(ctx, keys)
 	if err != nil {
 		return errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas")
@@ -231,7 +233,6 @@ func (manager *SQuotaBaseManager) __checkQuota(ctx context.Context, quota IQuota
 		if pendings[i].IsEmpty() {
 			continue
 		}
-		log.Debugf("__checkQuota pending %d: %s", i, jsonutils.Marshal(pendings[i]))
 		used.Add(pendings[i])
 	}
 	return used.Exceed(request, quota)
@@ -271,3 +272,51 @@ func (manager *SQuotaBaseManager) _checkSetPendingQuota(ctx context.Context, use
 	}
 	return nil
 }
+
+func (manager *SQuotaBaseManager) getQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
+	quotas, err := manager.GetParentQuotas(ctx, request.GetKeys())
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.getMatchedQuotas")
+	}
+	minCnt := -1
+	for i := len(quotas) - 1; i >= 0; i -= 1 {
+		rel := relation(quotas[i].GetKeys(), pendingKeys)
+		if rel == QuotaKeysContain || rel == QuotaKeysEqual {
+			break
+		}
+		cnt, err := manager.__getQuotaCount(ctx, quotas[i], request)
+		if err != nil {
+			return 0, errors.Wrapf(err, "manager.__getQuotaCount for key %s", QuotaKeyString(quotas[i].GetKeys()))
+		}
+		if minCnt < 0 || minCnt > cnt {
+			minCnt = cnt
+		}
+	}
+	return minCnt, nil
+}
+
+func (manager *SQuotaBaseManager) __getQuotaCount(ctx context.Context, quota IQuota, request IQuota) (int, error) {
+	keys := quota.GetKeys()
+	used := manager.newQuota()
+	err := manager.usageStore.GetQuota(ctx, keys, used)
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.usageStore.GetQuotaByKeys")
+	}
+	pendings, err := manager.pendingStore.GetChildrenQuotas(ctx, keys)
+	if err != nil {
+		return 0, errors.Wrap(err, "manager.pendingStore.GetChildrenQuotas")
+	}
+	for i := range pendings {
+		if pendings[i].IsEmpty() {
+			continue
+		}
+		used.Add(pendings[i])
+	}
+	err = used.Exceed(request, quota)
+	if err != nil {
+		return 0, nil
+	}
+	quota.Sub(used)
+	cnt := quota.Allocable(request)
+	return cnt, nil
+}

pkg/cloudcommon/db/quotas/register.go

@@ -20,6 +20,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -33,6 +34,8 @@ func init() {
 	quotaManagerTable = make(map[reflect.Type]IQuotaManager)
 
 	db.CancelUsages = CancelUsages
+	db.CancelPendingUsagesInContext = cancelPendingUsagesInContext
+	db.InitPendingUsagesInContext = initPendingUsagesInContext
 }
 
 func Register(manager IQuotaManager) {
@@ -59,7 +62,12 @@ func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential,
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
 	manager := getQuotaManager(quota)
-	return manager.checkSetPendingQuota(ctx, userCred, quota)
+	err := manager.checkSetPendingQuota(ctx, userCred, quota)
+	if err != nil {
+		return errors.Wrap(err, "manager.checkSetPendingQuota")
+	}
+	savePendingUsagesInContext(ctx, quota)
+	return nil
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
@@ -75,3 +83,8 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 		log.Errorf("cancelUsage %s fail: %s", jsonutils.Marshal(usage), err)
 	}
 }
+
+func GetQuotaCount(ctx context.Context, request IQuota, pendingKeys IQuotaKeys) (int, error) {
+	manager := getQuotaManager(request)
+	return manager.getQuotaCount(ctx, request, pendingKeys)
+}

pkg/cloudcommon/db/usages.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type IUsage interface {
+	FetchUsage(ctx context.Context) error
+	IsEmpty() bool
+}
+
+var (
+	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
+
+	CancelPendingUsagesInContext func(ctx context.Context, userCred mcclient.TokenCredential) error
+
+	InitPendingUsagesInContext func(ctx context.Context) context.Context
+)

pkg/cloudcommon/options/mergeconf.go

@@ -82,9 +82,12 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 		serviceConf, err := getServiceConfig(s.session, s.serviceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", serviceType, err)
-		} else {
+		} else if serviceConf != nil {
 			s.config.Update(serviceConf)
 			merged = true
+		} else {
+			// not initialized
+			s.Upload()
 		}
 	}
 	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
@@ -92,9 +95,11 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 		commonConf, err := getServiceConfig(s.session, commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
-		} else {
+		} else if commonConf != nil {
 			s.config.Update(commonConf)
 			merged = true
+		} else {
+			// common not initialized
 		}
 	}
 	if merged {

pkg/cloudcommon/workmanager/manager.go

@@ -65,6 +65,8 @@ func (w *SWorkManager) delayTask(ctx context.Context, task DelayTaskFunc, params
 		w.delayTaskWithoutReqctx(ctx, task, params, worker)
 		return
 	} else {
+		// delayTask should have a new context.Context with value 'taskid'
+		ctx = context.WithValue(context.Background(), appctx.APP_CONTEXT_KEY_TASK_ID, ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID))
 		w.add()
 		worker.Run(func() {
 			defer w.done()
@@ -109,6 +111,12 @@ func (w *SWorkManager) DelayTaskWithoutReqctx(ctx context.Context, task DelayTas
 func (w *SWorkManager) delayTaskWithoutReqctx(
 	ctx context.Context, task DelayTaskFunc, params interface{}, worker *appsrv.SWorkerManager,
 ) {
+	// delayTaskWithoutReqctx should have a new context.Context
+	newCtx := context.Background()
+	if ctx != nil && ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID) != nil {
+		newCtx = context.WithValue(newCtx, appctx.APP_CONTEXT_KEY_TASK_ID, ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID))
+	}
+	ctx = newCtx
 	w.add()
 	w.worker.Run(func() {
 		defer w.done()

pkg/cloudevent/models/cloudevents.go

@@ -16,34 +16,27 @@ package models
 
 import (
 	"context"
-	"database/sql"
-	"fmt"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/cloudevent"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudevent/options"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 )
 
 type SCloudeventManager struct {
-	db.SVirtualResourceBaseManager
+	db.SModelBaseManager
 }
 
 var CloudeventManager *SCloudeventManager
-var mods map[string]modulebase.Manager
 
 func init() {
 	CloudeventManager = &SCloudeventManager{
-		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+		SModelBaseManager: db.NewModelBaseManager(
 			SCloudevent{},
 			"cloudevents_tbl",
 			"cloudevent",
@@ -54,7 +47,7 @@ func init() {
 }
 
 type SCloudevent struct {
-	db.SVirtualResourceBase
+	db.SModelBase
 
 	Service      string               `width:"64" charset:"utf8" nullable:"true" list:"user"`
 	ResourceType string               `width:"64" charset:"utf8" nullable:"true" list:"user"`
@@ -63,8 +56,11 @@ type SCloudevent struct {
 	Request      jsonutils.JSONObject `charset:"utf8" nullable:"true" list:"user"`
 	Account      string               `width:"64" charset:"utf8" nullable:"true" list:"user"`
 	Success      bool                 `nullable:"false" list:"user"`
+	CreatedAt    time.Time            `nullable:"false" created_at:"true" index:"true" get:"user" list:"user"`
 
 	CloudproviderId string `width:"64" charset:"utf8" nullable:"true" list:"user"`
+	Manager         string `width:"128" charset:"utf8" nullable:"false" index:"true" list:"user"`
+	Provider        string `width:"64" charset:"ascii" nullable:"false" list:"user"`
 }
 
 func (self *SCloudeventManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
@@ -80,116 +76,20 @@ func (self *SCloudevent) AllowUpdateItem(ctx context.Context, userCred mcclient.
 }
 
 func (manager *SCloudeventManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input *api.CloudeventListInput) (*sqlchemy.SQuery, error) {
-	q, err := manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.JSON(input))
+	q, err := manager.SModelBaseManager.ListItemFilter(ctx, q, userCred, input.JSON(input))
 	if err != nil {
 		return nil, err
 	}
-	if len(input.Cloudprovider) > 0 {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(userCred, input.Cloudprovider)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), input.Cloudprovider)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		q = q.Equals("cloudprovider_id", providerObj.GetId())
-	}
 
 	if len(input.Providers) > 0 {
-		sq := CloudproviderManager.Query().SubQuery()
-		q = q.Join(sq, sqlchemy.Equals(q.Field("cloudprovider_id"), sq.Field("id"))).
-			Filter(sqlchemy.In(sq.Field("provider"), input.Providers))
+		q = q.In("provider", input.Providers)
 	}
-	//过滤已删除的cloudprovider日志
-	sq := CloudproviderManager.Query("id").SubQuery()
-	q = q.In("cloudprovider_id", sq)
+
 	return q, nil
 }
 
-func (self *SCloudevent) GetCloudprovider() (*SCloudprovider, error) {
-	cloudprovider, err := CloudproviderManager.FetchById(self.CloudproviderId)
-	if err != nil {
-		return nil, err
-	}
-	return cloudprovider.(*SCloudprovider), nil
-}
-
 func (self *SCloudevent) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
-	extra := self.SStatusStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query)
-	extra, _ = self.getMoreDetails(ctx, userCred, query, extra)
-	return extra
-}
-
-func (self *SCloudevent) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	cloudprovider, err := self.GetCloudprovider()
-	if err != nil {
-		return nil, err
-	}
-	info := jsonutils.Marshal(map[string]string{
-		"provider": cloudprovider.Provider,
-		"manager":  cloudprovider.Name,
-	})
-	extra.Update(info)
-	return extra, nil
-}
-
-func (self *SCloudevent) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) {
-	extra, err := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query)
-	if err != nil {
-		return nil, err
-	}
-	return self.getMoreDetails(ctx, userCred, query, extra)
-}
-
-func (self *SCloudeventManager) fetchMods(ctx context.Context, userCred mcclient.TokenCredential) {
-	if len(mods) > 0 {
-		return
-	}
-	s := auth.GetAdminSession(ctx, options.Options.Region, "v2")
-	mods = map[string]modulebase.Manager{}
-	rms, _ := modulebase.GetRegisterdModules()
-	for _, _mods := range rms {
-		for _, _mod := range _mods {
-			if _, ok := mods[_mod]; !ok {
-				mod, err := modulebase.GetModule(s, _mod)
-				if err != nil {
-					log.Errorf("failed to get mod %s error: %v", _mod, err)
-					continue
-				}
-				mods[mod.GetKeyword()] = mod
-			}
-		}
-	}
-	return
-}
-
-func (manager *SCloudeventManager) setEventInfo(session *mcclient.ClientSession, mod modulebase.Manager, event *SCloudevent) error {
-	params := jsonutils.NewDict()
-	params.Add(jsonutils.NewString(fmt.Sprintf("external_id.equals(%s)", event.Name)), "filter")
-	result, err := mod.List(session, params)
-	if err != nil {
-		return errors.Wrapf(err, "mod.List for %s by externalId: %s", mod.KeyString(), event.Name)
-	}
-	if len(result.Data) != 1 {
-		return errors.Wrapf(err, "found %d %s by externalId: %s", len(result.Data), mod.KeyString(), event.Name)
-	}
-
-	data := struct {
-		Name     string
-		TenantId string
-	}{}
-	err = result.Data[0].Unmarshal(&data)
-	if err != nil {
-		return errors.Wrapf(err, "result.Data[0].Unmarshal %s", result.Data[0])
-	}
-	if len(data.Name) > 0 {
-		event.Name = event.Name
-	}
-	if len(data.TenantId) > 0 {
-		event.ProjectId = data.TenantId
-	}
-	return nil
+	return self.SModelBase.GetCustomizeColumns(ctx, userCred, query)
 }
 
 func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider *SCloudprovider, iEvents []cloudprovider.ICloudEvent) int {
@@ -203,16 +103,12 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred
 			RequestId:       iEvent.GetRequestId(),
 			Request:         iEvent.GetRequest(),
 			Success:         iEvent.IsSuccess(),
+			Manager:         cloudprovider.Name,
+			Provider:        cloudprovider.Provider,
 			CloudproviderId: cloudprovider.Id,
 		}
 
-		event.Name = iEvent.GetName()
-		event.Status = "ready"
-		event.ProjectId = userCred.GetProjectId()
 		event.CreatedAt = iEvent.GetCreatedAt()
-		event.ProjectId = userCred.GetProjectId()
-		event.DomainId = userCred.GetDomainId()
-
 		event.SetModelManager(manager, event)
 		err := manager.TableSpec().Insert(event)
 		if err != nil {
@@ -223,3 +119,11 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred
 	}
 	return count
 }
+
+func (manager *SCloudeventManager) GetPagingConfig() *db.SPagingConfig {
+	return &db.SPagingConfig{
+		Order:        sqlchemy.SQL_ORDER_DESC,
+		MarkerField:  "created_at",
+		DefaultLimit: 20,
+	}
+}

pkg/cloudevent/models/cloudproviders.go

@@ -56,10 +56,11 @@ func init() {
 type SCloudprovider struct {
 	db.SEnabledStatusStandaloneResourceBase
 
-	HealthStatus  string `width:"16" charset:"ascii" default:"normal" nullable:"false" list:"domain"`
-	SyncStatus    string
-	LastSync      time.Time
-	LastSyncEndAt time.Time
+	HealthStatus   string `width:"16" charset:"ascii" default:"normal" nullable:"false" list:"domain"`
+	SyncStatus     string
+	LastSync       time.Time
+	LastSyncEndAt  time.Time
+	LastSyncTimeAt time.Time
 
 	AccessUrl string `width:"64" charset:"ascii" nullable:"true" list:"domain" update:"domain"`
 	Account   string `width:"128" charset:"ascii" nullable:"false" list:"domain"`
@@ -182,6 +183,18 @@ func (self *SCloudprovider) MarkEndSync(userCred mcclient.TokenCredential) error
 	return nil
 }
 
+func (self *SCloudprovider) SetLastSyncTimeAt(userCred mcclient.TokenCredential, last time.Time) error {
+	_, err := db.Update(self, func() error {
+		self.LastSyncTimeAt = last
+		return nil
+	})
+	if err != nil {
+		log.Errorf("Failed to SetLastSyncTimeAt error: %v", err)
+		return err
+	}
+	return nil
+}
+
 func (manager *SCloudproviderManager) newFromRegionProvider(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider SCloudprovider) error {
 	cloudprovider.SyncStatus = api.CLOUD_PROVIDER_SYNC_STATUS_IDLE
 	return manager.TableSpec().Insert(&cloudprovider)
@@ -226,19 +239,23 @@ func (self *SCloudprovider) GetNextTimeRange() (time.Time, time.Time, error) {
 	if err != nil {
 		return start, end, errors.Wrap(err, "q.CountWithError")
 	}
-	if count == 0 {
+	if !self.LastSyncTimeAt.IsZero() {
+		start = self.LastSyncTimeAt
+	} else if count == 0 {
 		start = time.Now().AddDate(0, 0, -1*factory.GetMaxCloudEventKeepDays())
 	} else {
-		provider := SCloudprovider{}
-		err = q.First(&provider)
+		event := &SCloudevent{}
+		err = q.First(event)
 		if err != nil {
 			return start, end, errors.Wrap(err, "q.First")
 		}
-		start = provider.CreatedAt
-		if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
-			start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)
-		}
+		start = event.CreatedAt
 	}
+	// 避免cloudevent过长时间未运行，再次运行时记录的最后一条时间距离现在间隔太长
+	if start.Before(time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)) {
+		start = time.Now().AddDate(0, 0, factory.GetMaxCloudEventKeepDays()*-1)
+	}
+
 	if options.Options.OneSyncForHours > factory.GetMaxCloudEventSyncDays()*24 {
 		end = start.Add(time.Duration(factory.GetMaxCloudEventSyncDays()*24) * time.Hour)
 	} else {

pkg/cloudevent/tasks/cloudevent_sync_task.go

@@ -97,6 +97,7 @@ func (self *CloudeventSyncTask) OnInit(ctx context.Context, obj db.IStandaloneMo
 		log.Infof("Sync %d events for %s(%s) from %s(%d) hours", _count, provider.Name, provider.Id, start.Format("2006-01-02T15:04:05Z"), duration/time.Hour)
 		count += _count
 
+		provider.SetLastSyncTimeAt(self.UserCred, end)
 		if time.Now().Sub(end) < duration {
 			break
 		}

pkg/cloudprovider/cloudprovider.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -115,6 +116,32 @@ type ICloudProvider interface {
 	GetCloudRegionExternalIdPrefix() string
 
 	GetStorageClasses(regionId string) []string
+
+	GetCapabilities() []string
+}
+
+func IsSupportProject(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_PROJECT, prod.GetCapabilities())
+}
+
+func IsSupportCompute(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_COMPUTE, prod.GetCapabilities())
+}
+
+func IsSupportLoadbalancer(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_LOADBALANCER, prod.GetCapabilities())
+}
+
+func IsSupportObjectstore(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_OBJECTSTORE, prod.GetCapabilities())
+}
+
+func IsSupportRds(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_RDS, prod.GetCapabilities())
+}
+
+func IsSupportElasticCache(prod ICloudProvider) bool {
+	return utils.IsInStringArray(CLOUD_CAPABILITY_CACHE, prod.GetCapabilities())
 }
 
 var providerTable map[string]ICloudProviderFactory

pkg/cloudprovider/consts.go

@@ -39,3 +39,13 @@ const (
 	ErrInvalidProvider     = httperrors.ErrInvalidProvider
 	ErrNoBalancePermission = httperrors.ErrNoBalancePermission
 )
+
+const (
+	CLOUD_CAPABILITY_PROJECT      = "project"
+	CLOUD_CAPABILITY_COMPUTE      = "compute"
+	CLOUD_CAPABILITY_LOADBALANCER = "loadbalancer"
+	CLOUD_CAPABILITY_OBJECTSTORE  = "objectstore"
+	CLOUD_CAPABILITY_RDS          = "rds"
+	CLOUD_CAPABILITY_CACHE        = "cache"
+	CLOUD_CAPABILITY_EVENT        = "event"
+)

pkg/cloudprovider/objectstore.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"net/http"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -276,6 +277,12 @@ func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) {
 	return stats, nil
 }
 
+type cloudObjectList []ICloudObject
+
+func (a cloudObjectList) Len() int           { return len(a) }
+func (a cloudObjectList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a cloudObjectList) Less(i, j int) bool { return a[i].GetKey() < a[j].GetKey() }
+
 func GetPagedObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool, marker string, maxCount int) ([]ICloudObject, string, error) {
 	delimiter := "/"
 	if isRecursive {
@@ -289,20 +296,22 @@ func GetPagedObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool,
 	if err != nil {
 		return nil, "", errors.Wrap(err, "bucket.ListObjects")
 	}
-	// Send all common prefixes if any.
-	// NOTE: prefixes are only present if the request is delimited.
-	if len(result.CommonPrefixes) > 0 {
-		ret = append(ret, result.CommonPrefixes...)
-	}
 	// Send all objects
 	for i := range result.Objects {
-		// if delimited, skip the first object
-		if !isRecursive && result.Objects[i].GetKey() == objectPrefix {
+		// if delimited, skip the first object ends with delimiter
+		if !isRecursive && result.Objects[i].GetKey() == objectPrefix && strings.HasSuffix(objectPrefix, delimiter) {
 			continue
 		}
 		ret = append(ret, result.Objects[i])
 		marker = result.Objects[i].GetKey()
 	}
+	// Send all common prefixes if any.
+	// NOTE: prefixes are only present if the request is delimited.
+	if len(result.CommonPrefixes) > 0 {
+		ret = append(ret, result.CommonPrefixes...)
+	}
+	// sort prefix by name in ascending order
+	sort.Sort(cloudObjectList(ret))
 	// If next marker present, save it for next request.
 	if result.NextMarker != "" {
 		marker = result.NextMarker

pkg/compute/guestdrivers/aliyun.go

@@ -52,7 +52,8 @@ func (self *SAliyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_ALIYUN
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_ALIYUN
+	keys.Hypervisor = api.HYPERVISOR_ALIYUN
 	return keys
 }
 

pkg/compute/guestdrivers/aws.go

@@ -99,7 +99,8 @@ func (self *SAwsGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, own
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_AWS
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_AWS
+	keys.Hypervisor = api.HYPERVISOR_AWS
 	return keys
 }
 

pkg/compute/guestdrivers/azure.go

@@ -53,7 +53,8 @@ func (self *SAzureGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, o
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_AZURE
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_AZURE
+	keys.Hypervisor = api.HYPERVISOR_AZURE
 	return keys
 }
 

pkg/compute/guestdrivers/baremetals.go

@@ -61,7 +61,7 @@ func (self *SBaremetalGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScop
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
 	keys.Provider = api.CLOUD_PROVIDER_ONECLOUD
-	// ignore brand
+	keys.Brand = api.ONECLOUD_BRAND_ONECLOUD
 	keys.Hypervisor = api.HYPERVISOR_BAREMETAL
 	return keys
 }

pkg/compute/guestdrivers/container.go

@@ -63,6 +63,7 @@ func (self *SContainerDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ow
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
 	keys.Provider = api.CLOUD_PROVIDER_ONECLOUD
+	keys.Brand = api.ONECLOUD_BRAND_ONECLOUD
 	keys.Hypervisor = api.HYPERVISOR_CONTAINER
 	return keys
 }

pkg/compute/guestdrivers/ctyun.go

@@ -44,7 +44,8 @@ func (self *SCtyunGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, o
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_CTYUN
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_CTYUN
+	keys.Hypervisor = api.HYPERVISOR_CTYUN
 	return keys
 }
 

pkg/compute/guestdrivers/esxi.go

@@ -65,8 +65,8 @@ func (self *SESXiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ow
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
 	keys.Provider = api.CLOUD_PROVIDER_VMWARE
-	// ignore brand
-	// ignore hypervisor keys.Hypervisor = api.HYPERVISOR_ESXI
+	keys.Brand = api.CLOUD_PROVIDER_VMWARE
+	keys.Hypervisor = api.HYPERVISOR_ESXI
 	return keys
 }
 

pkg/compute/guestdrivers/google.go

@@ -36,7 +36,8 @@ func (self *SGoogleGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_GOOGLE
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_GOOGLE
+	keys.Hypervisor = api.HYPERVISOR_GOOGLE
 	return keys
 }
 

pkg/compute/guestdrivers/huawei.go

@@ -50,7 +50,8 @@ func (self *SHuaweiGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_HUAWEI
-	// ignore brand
+	keys.Brand = api.CLOUD_PROVIDER_HUAWEI
+	keys.Hypervisor = api.HYPERVISOR_HUAWEI
 	return keys
 }
 

pkg/compute/guestdrivers/kvm.go

@@ -59,7 +59,7 @@ func (self *SKVMGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, own
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_ON_PREMISE
 	keys.Provider = api.CLOUD_PROVIDER_ONECLOUD
-	// ignore brand keys.Brand = brand
+	keys.Brand = api.ONECLOUD_BRAND_ONECLOUD
 	keys.Hypervisor = api.HYPERVISOR_KVM
 	return keys
 }

pkg/compute/guestdrivers/managedvirtual.go

@@ -1054,6 +1054,8 @@ func GetCloudVMStatus(vm cloudprovider.ICloudVM) string {
 		status = cloudprovider.CloudVMStatusChangeFlavor
 	case api.VM_DEPLOYING:
 		status = cloudprovider.CloudVMStatusDeploying
+	case api.VM_SUSPEND:
+		status = cloudprovider.CloudVMStatusSuspend
 	default:
 		status = cloudprovider.CloudVMStatusOther
 	}

pkg/compute/guestdrivers/openstack.go

@@ -69,6 +69,7 @@ func (self *SOpenStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScop
 	keys.CloudEnv = api.CLOUD_ENV_PRIVATE_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_OPENSTACK
 	keys.Brand = brand
+	keys.Hypervisor = api.HYPERVISOR_OPENSTACK
 	return keys
 }
 

pkg/compute/guestdrivers/qcloud.go

@@ -52,8 +52,8 @@ func (self *SQcloudGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_QCLOUD
-	// ignore brand keys.Brand = brand
-	// ignore hypervisor
+	keys.Brand = api.CLOUD_PROVIDER_QCLOUD
+	keys.Hypervisor = api.HYPERVISOR_QCLOUD
 	return keys
 }
 

pkg/compute/guestdrivers/ucloud.go

@@ -44,8 +44,8 @@ func (self *SUCloudGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.SBaseQuotaKeys = quotas.OwnerIdQuotaKeys(scope, ownerId)
 	keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_UCLOUD
-	// ignore brand
-	// ignore hypervisor
+	keys.Brand = api.CLOUD_PROVIDER_UCLOUD
+	keys.Hypervisor = api.HYPERVISOR_UCLOUD
 	return keys
 }
 

pkg/compute/guestdrivers/zstack.go

@@ -62,7 +62,7 @@ func (self *SZStackGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope,
 	keys.CloudEnv = api.CLOUD_ENV_PRIVATE_CLOUD
 	keys.Provider = api.CLOUD_PROVIDER_ZSTACK
 	keys.Brand = brand
-	// ignore hypervisor
+	keys.Hypervisor = api.HYPERVISOR_ZSTACK
 	return keys
 }
 

pkg/compute/hostdrivers/managedvirtual.go

@@ -33,6 +33,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -70,30 +71,29 @@ func (self *SManagedVirtualizationHostDriver) CheckAndSetCacheImage(ctx context.
 
 		cachedImage := scimg.GetCachedimage()
 		if cachedImage == nil {
-			return nil, fmt.Errorf("cached image not found???")
+			return nil, errors.Wrap(httperrors.ErrImageNotFound, "cached image not found???")
 		}
 
 		iStorageCache, err := storageCache.GetIStorageCache()
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "storageCache.GetIStorageCache")
 		}
 
 		image.ExternalId = scimg.ExternalId
 		if cachedImage.ImageType == cloudprovider.CachedImageTypeCustomized {
 			image.ExternalId, err = iStorageCache.UploadImage(ctx, userCred, image, isForce)
+			if err != nil {
+				return nil, errors.Wrap(err, "iStorageCache.UploadImage")
+			}
 		} else {
 			_, err = iStorageCache.GetIImageById(cachedImage.ExternalId)
 			if err != nil {
 				log.Errorf("remote image fetch error %s", err)
-				return nil, err
+				return nil, errors.Wrap(err, "iStorageCache.GetIImageById")
 			}
 			image.ExternalId = cachedImage.ExternalId
 		}
 
-		if err != nil {
-			return nil, err
-		}
-
 		// should record the externalId immediately
 		// so the waiting goroutine could pick the new externalId
 		// and avoid duplicate uploading
@@ -137,17 +137,17 @@ func (self *SManagedVirtualizationHostDriver) RequestUncacheImage(ctx context.Co
 
 		iImage, err := iStorageCache.GetIImageById(scimg.ExternalId)
 		if err != nil {
-			if err == cloudprovider.ErrNotFound {
+			if errors.Cause(err) == cloudprovider.ErrNotFound {
 				return nil, nil
 			}
 			log.Errorf("GetIImageById fail %s", err)
-			return nil, err
+			return nil, errors.Wrap(err, "iStorageCache.GetIImageById")
 		}
 
 		err = iImage.Delete(ctx)
 		if err != nil {
 			log.Errorf("iImage Delete fail %s", err)
-			return nil, err
+			return nil, errors.Wrap(err, "iImage.Delete")
 		}
 
 		return nil, nil

pkg/compute/models/capabilities.go

@@ -45,8 +45,8 @@ type SCapabilities struct {
 	ObjectStorageBrands         []string `json:",allowempty"`
 	DisabledObjectStorageBrands []string `json:",allowempty"`
 	ResourceTypes               []string `json:",allowempty"`
-	StorageTypes                []string `json:",allowempty"`
-	DataStorageTypes            []string `json:",allowempty"`
+	StorageTypes                []string `json:",allowempty"` // going to remove on 2.14
+	DataStorageTypes            []string `json:",allowempty"` // going to remove on 2.14
 	GPUModels                   []string `json:",allowempty"`
 	MinNicCount                 int
 	MaxNicCount                 int
@@ -57,6 +57,9 @@ type SCapabilities struct {
 	PublicNetworkCount          int
 	DBInstance                  map[string]map[string]map[string][]string //map[engine][engineVersion][category][]{storage_type}
 	Specs                       jsonutils.JSONObject
+
+	StorageTypes2     map[string][]string `json:",allowempty"`
+	DataStorageTypes2 map[string][]string `json:",allowempty"`
 }
 
 func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, region *SCloudregion, zone *SZone) (SCapabilities, error) {
@@ -88,8 +91,10 @@ func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, que
 	getBrands(region, zone, domainId, &capa)
 	// capa.Brands, capa.ComputeEngineBrands, capa.NetworkManageBrands, capa.ObjectStorageBrands = a, c, n, o
 	capa.ResourceTypes = getResourceTypes(region, zone, domainId)
-	capa.StorageTypes = getStorageTypes(region, zone, true, domainId)
-	capa.DataStorageTypes = getStorageTypes(region, zone, false, domainId)
+	capa.StorageTypes = getStorageTypes(region, zone, true, domainId, "")
+	capa.DataStorageTypes = getStorageTypes(region, zone, false, domainId, "")
+	capa.StorageTypes2 = getStorageTypes2(region, zone, true, domainId, capa.Hypervisors)
+	capa.DataStorageTypes2 = getStorageTypes2(region, zone, false, domainId, capa.Hypervisors)
 	capa.GPUModels = getGPUs(region, zone, domainId)
 	capa.SchedPolicySupport = isSchedPolicySupported(region, zone)
 	capa.MinNicCount = getMinNicCount(region, zone)
@@ -123,11 +128,11 @@ func getRegionZoneSubq(region *SCloudregion) *sqlchemy.SSubQuery {
 	return ZoneManager.Query("id").Equals("cloudregion_id", region.GetId()).SubQuery()
 }
 
-func getDomainManagerSubq(domainId string) *sqlchemy.SSubQuery {
+func domainManagerFieldFilter(domainId, field string) *sqlchemy.SSubQuery {
 	providers := CloudproviderManager.Query().SubQuery()
 	accounts := CloudaccountManager.Query().SubQuery()
 
-	q := providers.Query(providers.Field("id"))
+	q := providers.Query(providers.Field(field))
 	q = q.Join(accounts, sqlchemy.Equals(accounts.Field("id"), providers.Field("cloudaccount_id")))
 	q = q.Filter(sqlchemy.OR(
 		sqlchemy.AND(
@@ -146,6 +151,14 @@ func getDomainManagerSubq(domainId string) *sqlchemy.SSubQuery {
 	return q.SubQuery()
 }
 
+func getDomainManagerSubq(domainId string) *sqlchemy.SSubQuery {
+	return domainManagerFieldFilter(domainId, "id")
+}
+
+func getDomainManagerProviderSubq(domainId string) *sqlchemy.SSubQuery {
+	return domainManagerFieldFilter(domainId, "provider")
+}
+
 func getDBInstanceInfo(region *SCloudregion, zone *SZone) map[string]map[string]map[string][]string {
 	if zone != nil {
 		region = zone.GetRegion()
@@ -379,7 +392,16 @@ func getResourceTypes(region *SCloudregion, zone *SZone, domainId string) []stri
 	return resourceTypes
 }
 
-func getStorageTypes(region *SCloudregion, zone *SZone, isSysDisk bool, domainId string) []string {
+func getStorageTypes2(region *SCloudregion, zone *SZone, isSysDisk bool, domainId string, hypervisors []string) map[string][]string {
+	ret := make(map[string][]string)
+	for _, hypervisor := range hypervisors {
+		hostType := api.HYPERVISOR_HOSTTYPE[hypervisor]
+		ret[hypervisor] = getStorageTypes(region, zone, isSysDisk, domainId, hostType)
+	}
+	return ret
+}
+
+func getStorageTypes(region *SCloudregion, zone *SZone, isSysDisk bool, domainId string, hostType string) []string {
 	storages := StorageManager.Query().SubQuery()
 	hostStorages := HoststorageManager.Query().SubQuery()
 	hosts := HostManager.Query().SubQuery()
@@ -407,6 +429,9 @@ func getStorageTypes(region *SCloudregion, zone *SZone, isSysDisk bool, domainId
 			sqlchemy.IsNullOrEmpty(hosts.Field("manager_id")),
 		))
 	}
+	if len(hostType) > 0 {
+		q = q.Filter(sqlchemy.Equals(hosts.Field("host_type"), hostType))
+	}
 	q = q.Filter(sqlchemy.Equals(hosts.Field("resource_type"), api.HostResourceTypeShared))
 	q = q.Filter(sqlchemy.IsNotEmpty(storages.Field("storage_type")))
 	q = q.Filter(sqlchemy.IsNotNull(storages.Field("storage_type")))

pkg/compute/models/cloudaccounts.go

@@ -45,6 +45,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/choices"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -142,6 +143,18 @@ func (self *SCloudaccount) GetCloudproviders() []SCloudprovider {
 	return self.getCloudprovidersInternal(tristate.None)
 }
 
+func (self *SCloudaccount) IsAvailable() bool {
+	if !self.Enabled {
+		return false
+	}
+
+	if !utils.IsInStringArray(self.HealthStatus, api.CLOUD_PROVIDER_VALID_HEALTH_STATUS) {
+		return false
+	}
+
+	return true
+}
+
 func (self *SCloudaccount) GetEnabledCloudproviders() []SCloudprovider {
 	return self.getCloudprovidersInternal(tristate.True)
 }
@@ -163,9 +176,10 @@ func (self *SCloudaccount) getCloudprovidersInternal(enabled tristate.TriState)
 }
 
 func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error {
-	if self.EnableAutoSync {
-		return httperrors.NewInvalidStatusError("automatic syncing is enabled")
-	}
+	// allow delete cloudaccount if it is disabled
+	// if self.EnableAutoSync {
+	//	return httperrors.NewInvalidStatusError("automatic syncing is enabled")
+	// }
 	if self.Enabled {
 		return httperrors.NewInvalidStatusError("account is enabled")
 	}
@@ -333,7 +347,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		if cnt > 0 {
 			return input, httperrors.NewDuplicateResourceError("the account has been registerd %s", accountId)
 		}
-		data.Set("account_id", jsonutils.NewString(accountId))
+		input.AccountId = accountId
 	}
 
 	if input.SyncIntervalSeconds == 0 {
@@ -850,12 +864,14 @@ func (self *SCloudaccount) getProjectIds() []string {
 		if err != nil {
 			return nil
 		}
-		ret = append(ret, projId)
+		if len(projId) > 0 && utils.IsInStringArray(projId, ret) {
+			ret = append(ret, projId)
+		}
 	}
 	return ret
 }
 
-func (self *SCloudaccount) getCloudEnv() string {
+func (self *SCloudaccount) GetCloudEnv() string {
 	if self.IsOnPremise {
 		return api.CLOUD_ENV_ON_PREMISE
 	} else if self.IsPublicCloud.IsTrue() {
@@ -885,7 +901,7 @@ func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.
 	extra.Add(projects, "projects")
 	extra.Set("sync_interval_seconds", jsonutils.NewInt(int64(self.getSyncIntervalSeconds())))
 	extra.Set("sync_status2", jsonutils.NewString(self.getSyncStatus2()))
-	extra.Set("cloud_env", jsonutils.NewString(self.getCloudEnv()))
+	extra.Set("cloud_env", jsonutils.NewString(self.GetCloudEnv()))
 	if len(self.ProjectId) > 0 {
 		if proj, _ := db.TenantCacheManager.FetchTenantById(context.Background(), self.ProjectId); proj != nil {
 			extra.Add(jsonutils.NewString(proj.Name), "tenant")
@@ -1694,3 +1710,80 @@ func (manager *SCloudaccountManager) FilterByOwner(q *sqlchemy.SQuery, owner mcc
 	}
 	return q
 }
+
+func (manager *SCloudaccountManager) getBrandsOfProvider(provider string) ([]string, error) {
+	q := manager.Query().Equals("provider", provider)
+	cloudaccounts := make([]SCloudaccount, 0)
+	err := db.FetchModelObjects(manager, q, &cloudaccounts)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	ret := make([]string, 0)
+	for i := range cloudaccounts {
+		if cloudaccounts[i].IsAvailable() && !utils.IsInStringArray(cloudaccounts[i].Brand, ret) {
+			ret = append(ret, cloudaccounts[i].Brand)
+		}
+	}
+	return ret, nil
+}
+
+func guessBrandForHypervisor(hypervisor string) string {
+	driver := GetDriver(hypervisor)
+	if driver == nil {
+		log.Errorf("guestBrandFromHypervisor: fail to find driver for hypervisor %s", hypervisor)
+		return ""
+	}
+	provider := driver.GetProvider()
+	if len(provider) == 0 {
+		log.Errorf("guestBrandFromHypervisor: fail to find provider for hypervisor %s", hypervisor)
+		return ""
+	}
+	brands, err := CloudaccountManager.getBrandsOfProvider(provider)
+	if err != nil {
+		log.Errorf("guestBrandFromHypervisor: fail to find brands for hypervisor %s", hypervisor)
+		return ""
+	}
+	if len(brands) != 1 {
+		log.Errorf("guestBrandFromHypervisor: find mistached number of brands for hypervisor %s %s", hypervisor, brands)
+		return ""
+	}
+	return brands[0]
+}
+
+func (account *SCloudaccount) AllowPerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return db.IsAllowPerform(rbacutils.ScopeSystem, userCred, account, "sync-skus")
+}
+
+func (account *SCloudaccount) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if !account.Enabled {
+		return nil, httperrors.NewInvalidStatusError("Account disabled")
+	}
+
+	dataDict := data.(*jsonutils.JSONDict)
+	resourceV := validators.NewStringChoicesValidator("resource", choices.NewChoices(ServerSkuManager.Keyword(), ElasticcacheSkuManager.Keyword(), DBInstanceSkuManager.Keyword()))
+	regionV := validators.NewModelIdOrNameValidator("cloudregion", "cloudregion", account.GetOwnerId())
+	providerV := validators.NewModelIdOrNameValidator("cloudprovider", "cloudprovider", account.GetOwnerId())
+	keyV := map[string]validators.IValidator{
+		"resource":      resourceV,
+		"cloudregion":   regionV.Optional(true),
+		"cloudprovider": providerV.Optional(true),
+	}
+
+	for _, v := range keyV {
+		if err := v.Validate(dataDict); err != nil {
+			return nil, err
+		}
+	}
+
+	force, _ := data.Bool("force")
+	if account.CanSync() || force {
+		task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountSyncSkusTask", account, userCred, dataDict, "", "", nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "CloudAccountSyncSkusTask")
+		}
+
+		task.ScheduleRun(nil)
+	}
+
+	return nil, nil
+}

pkg/compute/models/cloudproviders.go

@@ -109,10 +109,11 @@ func (self *SCloudprovider) AllowDeleteItem(ctx context.Context, userCred mcclie
 }
 
 func (self *SCloudprovider) ValidateDeleteCondition(ctx context.Context) error {
-	account := self.GetCloudaccount()
-	if account != nil && account.EnableAutoSync {
-		return httperrors.NewInvalidStatusError("auto syncing is enabled on account")
-	}
+	// allow delete cloudprovider if it is disabled
+	// account := self.GetCloudaccount()
+	// if account != nil && account.EnableAutoSync {
+	// 	return httperrors.NewInvalidStatusError("auto syncing is enabled on account")
+	// }
 	if self.Enabled {
 		return httperrors.NewInvalidStatusError("provider is enabled")
 	}
@@ -1152,7 +1153,7 @@ func (self *SCloudprovider) IsAvailable() bool {
 	if !utils.IsInStringArray(self.Status, api.CLOUD_PROVIDER_VALID_STATUS) {
 		return false
 	}
-	if self.HealthStatus != api.CLOUD_PROVIDER_HEALTH_NORMAL {
+	if !utils.IsInStringArray(self.HealthStatus, api.CLOUD_PROVIDER_VALID_HEALTH_STATUS) {
 		return false
 	}
 	return true

pkg/compute/models/cloudsync.go

@@ -113,7 +113,7 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca
 
 	if cnt == 0 {
 		// 提前同步instance type.如果同步失败可能导致vm 内存显示为0
-		if err = syncServerSkusByRegion(ctx, userCred, localRegion); err != nil {
+		if err = SyncServerSkusByRegion(ctx, userCred, localRegion); err != nil {
 			msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), err)
 			log.Errorln(msg)
 			// 暂时不终止同步
@@ -135,7 +135,7 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca
 		}
 
 		if cnt == 0 {
-			syncElasticCacheSkusByRegion(ctx, userCred, localRegion)
+			SyncElasticCacheSkusByRegion(ctx, userCred, localRegion)
 		}
 	}
 }
@@ -995,58 +995,72 @@ func syncPublicCloudProviderInfo(
 
 	storageCachePairs := make([]sStoragecacheSyncPair, 0)
 
-	syncProjects(ctx, userCred, syncResults, driver, provider)
+	if cloudprovider.IsSupportProject(driver) {
+		syncProjects(ctx, userCred, syncResults, driver, provider)
+	}
 
 	localZones, remoteZones, _ := syncRegionZones(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
 
 	if !driver.GetFactory().NeedSyncSkuFromCloud() {
 		syncRegionSkus(ctx, userCred, localRegion)
-		syncRegionDBInstanceSkus(ctx, userCred, localRegion.Id, true)
+		SyncRegionDBInstanceSkus(ctx, userCred, localRegion.Id, true)
 	} else {
 		syncSkusFromPrivateCloud(ctx, userCred, syncResults, provider, remoteRegion)
 	}
 
 	// no need to lock public cloud region as cloud region for public cloud is readonly
 
-	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
-
-	// 需要先同步vpc，避免私有云eip找不到network
-	syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-
-	syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-	// sync snapshot policies before sync disks
-	syncRegionSnapshotPolicies(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-
-	for j := 0; j < len(localZones); j += 1 {
-
-		if len(syncRange.Zone) > 0 && !utils.IsInStringArray(localZones[j].Id, syncRange.Zone) {
-			continue
-		}
-		// no need to lock zone as public cloud zone is read-only
-
-		newPairs := syncZoneStorages(ctx, userCred, syncResults, provider, driver, &localZones[j], remoteZones[j], syncRange, storageCachePairs)
-		if len(newPairs) > 0 {
-			storageCachePairs = append(storageCachePairs, newPairs...)
-		}
-		newPairs = syncZoneHosts(ctx, userCred, syncResults, provider, driver, &localZones[j], remoteZones[j], syncRange, storageCachePairs)
-		if len(newPairs) > 0 {
-			storageCachePairs = append(storageCachePairs, newPairs...)
-		}
+	if cloudprovider.IsSupportObjectstore(driver) {
+		syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
 	}
 
-	// sync snapshots after sync disks
-	syncRegionSnapshots(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	if cloudprovider.IsSupportCompute(driver) {
+		// 需要先同步vpc，避免私有云eip找不到network
+		syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 
-	syncRegionLoadbalancerAcls(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-	syncRegionLoadbalancerCertificates(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-	syncRegionLoadbalancers(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		// sync snapshot policies before sync disks
+		syncRegionSnapshotPolicies(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 
-	syncRegionNetworkInterfaces(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		for j := 0; j < len(localZones); j += 1 {
 
-	syncRegionDBInstances(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
-	syncRegionDBInstanceBackups(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+			if len(syncRange.Zone) > 0 && !utils.IsInStringArray(localZones[j].Id, syncRange.Zone) {
+				continue
+			}
+			// no need to lock zone as public cloud zone is read-only
 
-	syncElasticcaches(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+			newPairs := syncZoneStorages(ctx, userCred, syncResults, provider, driver, &localZones[j], remoteZones[j], syncRange, storageCachePairs)
+			if len(newPairs) > 0 {
+				storageCachePairs = append(storageCachePairs, newPairs...)
+			}
+			newPairs = syncZoneHosts(ctx, userCred, syncResults, provider, driver, &localZones[j], remoteZones[j], syncRange, storageCachePairs)
+			if len(newPairs) > 0 {
+				storageCachePairs = append(storageCachePairs, newPairs...)
+			}
+		}
+
+		// sync snapshots after sync disks
+		syncRegionSnapshots(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	}
+
+	if cloudprovider.IsSupportLoadbalancer(driver) {
+		syncRegionLoadbalancerAcls(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		syncRegionLoadbalancerCertificates(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		syncRegionLoadbalancers(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	}
+
+	if cloudprovider.IsSupportCompute(driver) {
+		syncRegionNetworkInterfaces(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	}
+
+	if cloudprovider.IsSupportRds(driver) {
+		syncRegionDBInstances(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		syncRegionDBInstanceBackups(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	}
+
+	if cloudprovider.IsSupportElasticCache(driver) {
+		syncElasticcaches(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+	}
 
 	log.Debugf("storageCachePairs count %d", len(storageCachePairs))
 	for i := range storageCachePairs {
@@ -1074,7 +1088,9 @@ func syncOnPremiseCloudProviderInfo(
 ) error {
 	log.Debugf("Start sync on-premise provider %s(%s)", provider.Name, provider.Provider)
 
-	syncProjects(ctx, userCred, syncResults, driver, provider)
+	if cloudprovider.IsSupportProject(driver) {
+		syncProjects(ctx, userCred, syncResults, driver, provider)
+	}
 
 	iregion, err := driver.GetOnPremiseIRegion()
 	if err != nil {
@@ -1084,41 +1100,39 @@ func syncOnPremiseCloudProviderInfo(
 	}
 
 	localRegion := CloudregionManager.FetchDefaultRegion()
-	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion)
 
-	ihosts, err := iregion.GetIHosts()
-	if err != nil {
-		msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err)
-		log.Errorf(msg)
-		return err
+	if cloudprovider.IsSupportObjectstore(driver) {
+		syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion)
 	}
 
-	localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts)
-
-	syncResults.Add(HostManager, result)
-
-	msg := result.Result()
-	notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg)
-	log.Infof(notes)
-	//if result.IsError() {
-	//	logSyncFailed(provider, task, msg)
-	// return
-	//}
-	// db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
-	// logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
-
 	storageCachePairs := make([]sStoragecacheSyncPair, 0)
+	if cloudprovider.IsSupportCompute(driver) {
+		ihosts, err := iregion.GetIHosts()
+		if err != nil {
+			msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err)
+			log.Errorf(msg)
+			return err
+		}
 
-	for i := 0; i < len(localHosts); i += 1 {
-		if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) {
-			continue
+		localHosts, remoteHosts, result := HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts)
+
+		syncResults.Add(HostManager, result)
+
+		msg := result.Result()
+		notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg)
+		log.Infof(notes)
+
+		for i := 0; i < len(localHosts); i += 1 {
+			if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) {
+				continue
+			}
+			newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs)
+			if len(newCachePairs) > 0 {
+				storageCachePairs = append(storageCachePairs, newCachePairs...)
+			}
+			syncHostNics(ctx, userCred, provider, &localHosts[i], remoteHosts[i])
+			syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange)
 		}
-		newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs)
-		if len(newCachePairs) > 0 {
-			storageCachePairs = append(storageCachePairs, newCachePairs...)
-		}
-		syncHostNics(ctx, userCred, provider, &localHosts[i], remoteHosts[i])
-		syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange)
 	}
 
 	log.Debugf("storageCachePairs count %d", len(storageCachePairs))
@@ -1131,6 +1145,7 @@ func syncOnPremiseCloudProviderInfo(
 		log.Infof("syncCloudImages result: %s", msg)
 		// }
 	}
+
 	return nil
 }
 

pkg/compute/models/dbinstance_skus.go

@@ -114,6 +114,12 @@ func (manager *SDBInstanceSkuManager) ListItemFilter(ctx context.Context, q *sql
 		return nil, err
 	}
 	data := query.(*jsonutils.JSONDict)
+
+	if domian_id, _ := data.GetString("domain_id"); len(domian_id) > 0 {
+		data.Remove("domain_id")
+		q = q.In("provider", getDomainManagerProviderSubq(domian_id))
+	}
+
 	return validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{
 		{Key: "cloudregion", ModelKeyword: "cloudregion", OwnerId: userCred},
 	})
@@ -331,7 +337,7 @@ func (manager *SDBInstanceSkuManager) GetDBInstanceSkus(provider, cloudregionId,
 	return skus, nil
 }
 
-func (manager *SDBInstanceSkuManager) syncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, meta *SSkuResourcesMeta) compare.SyncResult {
+func (manager *SDBInstanceSkuManager) SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, meta *SSkuResourcesMeta) compare.SyncResult {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -451,7 +457,7 @@ func (manager *SDBInstanceSkuManager) newFromCloudSku(ctx context.Context, userC
 	return manager.TableSpec().Insert(sku)
 }
 
-func syncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart bool) {
+func SyncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart bool) {
 	if isStart {
 		q := DBInstanceSkuManager.Query()
 		if len(regionId) > 0 {
@@ -480,7 +486,7 @@ func syncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCreden
 		return
 	}
 
-	meta, err := fetchSkuResourcesMeta()
+	meta, err := FetchSkuResourcesMeta()
 	if err != nil {
 		log.Errorf("failed to fetch sku resource meta: %v", err)
 		return
@@ -491,7 +497,7 @@ func syncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCreden
 			log.Infof("region %s(%s) not support dbinstance, skip sync", region.Name, region.Id)
 			continue
 		}
-		result := DBInstanceSkuManager.syncDBInstanceSkus(ctx, userCred, &region, meta)
+		result := DBInstanceSkuManager.SyncDBInstanceSkus(ctx, userCred, &region, meta)
 		msg := result.Result()
 		notes := fmt.Sprintf("SyncDBInstanceSkus for region %s result: %s", region.Name, msg)
 		log.Infof(notes)
@@ -500,5 +506,5 @@ func syncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCreden
 }
 
 func SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
-	syncRegionDBInstanceSkus(ctx, userCred, "", isStart)
+	SyncRegionDBInstanceSkus(ctx, userCred, "", isStart)
 }

pkg/compute/models/dbinstances.go

@@ -41,6 +41,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/seclib2"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
@@ -182,6 +183,12 @@ func (man *SDBInstanceManager) ValidateCreateData(ctx context.Context, userCred
 		return nil, errors.Wrapf(err, "Unmarshal input failed: %v", err)
 	}
 
+	if len(input.Password) > 0 {
+		if !seclib2.MeetComplxity(input.Password) {
+			return nil, httperrors.NewWeakPasswordError()
+		}
+	}
+
 	network := networkV.Model.(*SNetwork)
 	input.NetworkExternalId = network.ExternalId
 

pkg/compute/models/disks.go

@@ -216,14 +216,11 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 			return nil, err
 		}
 		guest := iGuest.(*SGuest)
-		hoststorages := HoststorageManager.Query().SubQuery()
-		q = q.Join(hoststorages, sqlchemy.AND(
-			sqlchemy.Equals(hoststorages.Field("host_id"), guest.HostId),
-			sqlchemy.IsFalse(hoststorages.Field("deleted")))).
-			Join(storages, sqlchemy.AND(
-				sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")),
-				sqlchemy.IsFalse(storages.Field("deleted")))).
-			Filter(sqlchemy.Equals(storages.Field("id"), q.Field("storage_id")))
+		guestDisks := GuestdiskManager.Query().SubQuery()
+		q = q.Join(guestDisks, sqlchemy.AND(
+			sqlchemy.Equals(guestDisks.Field("disk_id"), q.Field("id")),
+			sqlchemy.Equals(guestDisks.Field("guest_id"), guest.Id),
+		))
 	}
 
 	storageStr := jsonutils.GetAnyString(queryDict, []string{"storage", "storage_id"})
@@ -407,10 +404,11 @@ func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok
 
 func diskCreateInput2ComputeQuotaKeys(input api.DiskCreateInput, ownerId mcclient.IIdentityProvider) SComputeResourceKeys {
 	// input.Hypervisor must be set
+	brand := guessBrandForHypervisor(input.Hypervisor)
 	keys := GetDriver(input.Hypervisor).GetComputeQuotaKeys(
 		rbacutils.ScopeProject,
 		ownerId,
-		"",
+		brand,
 	)
 	if len(input.PreferHost) > 0 {
 		hostObj, _ := HostManager.FetchById(input.PreferHost)
@@ -1037,6 +1035,18 @@ func (self *SDisk) StartDiskSaveTask(ctx context.Context, userCred mcclient.Toke
 }
 
 func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error {
+	provider := self.GetCloudprovider()
+	if provider != nil {
+		if !provider.IsAvailable() {
+			return httperrors.NewNotSufficientPrivilegeError("cloud provider %s is not available", provider.GetName())
+		}
+
+		account := provider.GetCloudaccount()
+		if account != nil && !account.IsAvailable() {
+			return httperrors.NewNotSufficientPrivilegeError("cloud account %s is not available", account.GetName())
+		}
+	}
+
 	return self.validateDeleteCondition(ctx, false)
 }
 
@@ -1059,6 +1069,18 @@ func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) er
 }
 
 func (self *SDisk) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	provider := self.GetCloudprovider()
+	if provider != nil {
+		if !provider.IsAvailable() {
+			return false
+		}
+
+		account := provider.GetCloudaccount()
+		if account != nil && !account.IsAvailable() {
+			return false
+		}
+	}
+
 	overridePendingDelete := false
 	purge := false
 	if query != nil {
@@ -2455,3 +2477,96 @@ func (disk *SDisk) GetUsages() []db.IUsage {
 		&usage,
 	}
 }
+
+func (disk *SDisk) AllowPerformBindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject) bool {
+
+	return disk.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, disk, "bind-snapshotpolicy")
+}
+
+func (disk *SDisk) PerformBindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+
+	spIden, err := data.GetString("snapshotpolicy")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("miss snapshotpolicy")
+	}
+	// check snapshotpolicy
+	imodel, err := db.FetchByIdOrName(SnapshotPolicyManager, userCred, spIden)
+	if errors.Cause(err) == sql.ErrNoRows {
+		return nil, httperrors.NewInputParameterError("no such snapshotpolicy %s", spIden)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchByIdOrName")
+	}
+	snapshotpolicy := imodel.(*SSnapshotPolicy)
+
+	// try to bind
+	spd, err := SnapshotPolicyDiskManager.newSnapshotpolicyDisk(ctx, userCred, snapshotpolicy, disk)
+
+	if errors.Cause(err) == ErrExistSD {
+		if spd.Status != api.SNAPSHOT_POLICY_DISK_INIT {
+			return nil, nil
+		}
+	} else if err != nil {
+		return nil, errors.Wrap(err, "SnapshotPolicyDiskManager.newSnapshotpolicyDisk")
+	}
+
+	// start up SnapshotPolicyApplyTask
+	taskData := jsonutils.NewDict()
+	taskData.Add(jsonutils.Marshal(spd), "snapshotPolicyDisk")
+	taskData.Add(jsonutils.Marshal(snapshotpolicy), "snapshotPolicy")
+	if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyApplyTask", disk, userCred, nil, "", "",
+		nil); err != nil {
+		return nil, errors.Wrap(err, "fail to start up SnapshotPolicyApplyTask")
+	} else {
+		task.ScheduleRun(taskData)
+	}
+	return nil, nil
+}
+
+func (disk *SDisk) AllowPerformUnbindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject) bool {
+
+	return disk.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, disk, "unbind-snapshotpolicy")
+}
+
+func (disk *SDisk) PerformUnbindSnapshotpolicy(ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+
+	spIden, err := data.GetString("snapshotpolicy")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("miss snapshotpolicy")
+	}
+	// check snapshotpolicy
+	imodel, err := db.FetchByIdOrName(SnapshotPolicyManager, userCred, spIden)
+	if errors.Cause(err) == sql.ErrNoRows {
+		return nil, httperrors.NewInputParameterError("no such snapshotpolicy %s", spIden)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "db.FetchByIdOrName")
+	}
+	snapshotpolicy := imodel.(*SSnapshotPolicy)
+
+	spd, err := SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk(snapshotpolicy.GetId(), disk.GetId())
+	if err != nil {
+		return nil, errors.Wrap(err, "SnapshotPolicyDiskManager.FetchBySnapshotPolicyDisk")
+	}
+	if spd == nil {
+		// has been detach
+		return nil, nil
+	}
+
+	// start up SnapshotPolicyCancelTask
+	taskdata := jsonutils.NewDict()
+	taskdata.Add(jsonutils.NewString(snapshotpolicy.Id), "snapshot_policy_id")
+	taskdata.Add(jsonutils.Marshal(spd), "snapshotPolicyDisk")
+	if task, err := taskman.TaskManager.NewTask(ctx, "SnapshotPolicyCancelTask", disk, userCred, nil, "", "",
+		nil); err != nil {
+		return nil, errors.Wrap(err, "fail to start up SnapshotPolicyCancelTask")
+	} else {
+		spd.SetStatus(userCred, api.SNAPSHOT_POLICY_DISK_DELETING, "")
+		task.ScheduleRun(taskdata)
+	}
+	return nil, nil
+}

pkg/compute/models/dnsrecords.go

@@ -375,6 +375,26 @@ func (man *SDnsRecordManager) QueryDnsIps(projectId, name, kind string) []*DnsIp
 	return dnsIps
 }
 
+func (rec *SDnsRecord) IsCNAME() bool {
+	return strings.HasPrefix(rec.Records, "CNAME:")
+}
+
+func (rec *SDnsRecord) HasRecordType(typ string) bool {
+	for _, r := range rec.GetInfo() {
+		if strings.HasPrefix(r, typ+":") {
+			return true
+		}
+	}
+	return false
+}
+
+func (rec *SDnsRecord) GetCNAME() string {
+	if !rec.IsCNAME() {
+		panic("not a cname record: " + rec.Records)
+	}
+	return rec.Records[len("CNAME:"):]
+}
+
 func (rec *SDnsRecord) GetInfo() []string {
 	return strings.Split(rec.Records, DNS_RECORDS_SEPARATOR)
 }

pkg/compute/models/elasticcache_accounts.go

@@ -233,6 +233,12 @@ func (manager *SElasticcacheAccountManager) ValidateCreateData(ctx context.Conte
 	}
 	data.Update(jsonutils.Marshal(input))
 
+	passwd, _ := data.GetString("password")
+	if reset, _ := data.Bool("reset_password"); reset && len(passwd) == 0 {
+		passwd = seclib2.RandomPassword2(12)
+		data.Set("password", jsonutils.NewString(passwd))
+	}
+
 	return region.GetDriver().ValidateCreateElasticcacheAccountData(ctx, userCred, ownerId, data)
 }
 

pkg/compute/models/elasticcache_skus.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -140,6 +141,24 @@ func (manager *SElasticcacheSkuManager) FetchCustomizeColumns(ctx context.Contex
 
 		fileds := jsonutils.NewDict()
 		fileds.Set("region", jsonutils.NewString(regions[cloudregionId]))
+		if region, err := db.FetchById(CloudregionManager, cloudregionId); err == nil {
+			fileds.Set("region_external_id", jsonutils.NewString(region.(*SCloudregion).ExternalId))
+			segs := strings.Split(region.(*SCloudregion).ExternalId, "/")
+			if len(segs) >= 2 {
+				fileds.Set("region_ext_id", jsonutils.NewString(segs[1]))
+			}
+		}
+
+		zoneId := objs[i].(*SElasticcacheSku).ZoneId
+		if len(zoneId) > 0 {
+			if zone, err := db.FetchById(ZoneManager, zoneId); err == nil {
+				fileds.Set("zone_external_id", jsonutils.NewString(zone.(*SZone).ExternalId))
+				segs := strings.Split(zone.(*SZone).ExternalId, "/")
+				if len(segs) >= 3 {
+					fileds.Set("zone_ext_id", jsonutils.NewString(segs[2]))
+				}
+			}
+		}
 
 		ret = append(ret, fileds)
 	}
@@ -177,6 +196,11 @@ func (manager *SElasticcacheSkuManager) ListItemFilter(ctx context.Context, q *s
 		return nil, err
 	}
 
+	if domian_id, _ := data.GetString("domain_id"); len(domian_id) > 0 {
+		data.Remove("domain_id")
+		q = q.In("provider", getDomainManagerProviderSubq(domian_id))
+	}
+
 	city, _ := query.GetString("city")
 	if len(city) > 0 {
 		regionTable := CloudregionManager.Query().SubQuery()
@@ -210,7 +234,7 @@ func (manager *SElasticcacheSkuManager) FetchSkusByRegion(regionID string) ([]SE
 	return skus, nil
 }
 
-func (manager *SElasticcacheSkuManager) syncElasticcacheSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult {
+func (manager *SElasticcacheSkuManager) SyncElasticcacheSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -415,3 +439,32 @@ func (manager *SElasticcacheSkuManager) GetPropertyCapability(ctx context.Contex
 
 	return result, nil
 }
+
+func (manager *SElasticcacheSkuManager) PerformActionSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	data := query.(*jsonutils.JSONDict)
+	cloudprovider := validators.NewModelIdOrNameValidator("cloudprovider", "cloudprovider", nil)
+	cloudregion := validators.NewModelIdOrNameValidator("cloudregion", "cloudregion", nil)
+
+	keyV := map[string]validators.IValidator{
+		"provider":    cloudprovider.Optional(true),
+		"cloudregion": cloudregion.Optional(true),
+	}
+
+	for _, v := range keyV {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+
+	regions := []SCloudregion{}
+	if region, err := data.GetString("cloudregion"); err == nil && len(region) > 0 {
+		regions = append(regions, *cloudregion.Model.(*SCloudregion))
+	} else if provider, err := data.GetString("cloudprovider"); err == nil && len(provider) > 0 {
+		regions, err = CloudregionManager.GetRegionByProvider(provider)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return nil, nil
+}

pkg/compute/models/elasticips.go

@@ -437,7 +437,7 @@ func (manager *SElasticipManager) newFromCloudEip(ctx context.Context, userCred
 }
 
 func (manager *SElasticipManager) getEipForInstance(instanceType string, instanceId string) (*SElasticip, error) {
-	return manager.getEip(instanceType, instanceId, "")
+	return manager.getEip(instanceType, instanceId, api.EIP_MODE_STANDALONE_EIP)
 }
 
 func (manager *SElasticipManager) getEip(instanceType string, instanceId string, eipMode string) (*SElasticip, error) {
@@ -875,6 +875,18 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient.
 		return nil, httperrors.NewInvalidStatusError("cannot associate server in status %s", server.Status)
 	}
 
+	if len(self.NetworkId) > 0 {
+		gns, err := server.GetNetworks("")
+		if err != nil {
+			return nil, httperrors.NewGeneralError(errors.Wrap(err, "GetNetworks"))
+		}
+		for _, gn := range gns {
+			if gn.NetworkId == self.NetworkId {
+				return nil, httperrors.NewInputParameterError("cannot associate eip with same network")
+			}
+		}
+	}
+
 	serverRegion := server.getRegion()
 	if serverRegion == nil {
 		return nil, httperrors.NewInputParameterError("server region is not found???")
@@ -1103,7 +1115,13 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 	eip.CloudregionId = region.Id
 	eip.Name = fmt.Sprintf("eip-for-%s", vm.GetName())
 
-	err := manager.TableSpec().Insert(&eip)
+	var err error
+	eip.Name, err = db.GenerateName(manager, userCred, eip.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.GenerateName")
+	}
+
+	err = manager.TableSpec().Insert(&eip)
 	if err != nil {
 		log.Errorf("create EIP record fail %s", err)
 		return nil, err

pkg/compute/models/groups.go

@@ -91,7 +91,8 @@ func (group *SGroup) GetCustomizeColumns(ctx context.Context, userCred mcclient.
 	query jsonutils.JSONObject) *jsonutils.JSONDict {
 	extra := group.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
 	ret, _ := group.getMoreDetails(ctx, userCred, extra)
-	return ret.JSON(ret)
+	extra.Update(ret.JSON(ret))
+	return extra
 }
 
 func (group *SGroup) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential,

pkg/compute/models/guest_actions.go

@@ -2090,21 +2090,21 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		if err != nil {
 			return nil, err
 		}
-		var inicCnt, enicCnt, ibw, ebw int
-		if isExitNetworkInfo(conf) {
+		var inicCnt, enicCnt int
+		if IsExitNetworkInfo(conf) {
 			enicCnt = 1
-			ebw = conf.BwLimit
+			// ebw = conf.BwLimit
 		} else {
 			inicCnt = 1
-			ibw = conf.BwLimit
+			// ibw = conf.BwLimit
 		}
 		pendingUsage := &SRegionQuota{
 			Port:  inicCnt,
 			Eport: enicCnt,
-			Bw:    ibw,
-			Ebw:   ebw,
+			//Bw:    ibw,
+			//Ebw:   ebw,
 		}
-		keys, err := self.GetQuotaKeys()
+		keys, err := self.GetRegionalQuotaKeys()
 		if err != nil {
 			return nil, err
 		}
@@ -2808,6 +2808,18 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 		return nil, httperrors.NewInputParameterError("cannot associate eip and instance in different region")
 	}
 
+	if len(eip.NetworkId) > 0 {
+		gns, err := self.GetNetworks("")
+		if err != nil {
+			return nil, httperrors.NewGeneralError(errors.Wrap(err, "GetNetworks"))
+		}
+		for _, gn := range gns {
+			if gn.NetworkId == eip.NetworkId {
+				return nil, httperrors.NewInputParameterError("cannot associate eip with same network")
+			}
+		}
+	}
+
 	eipZone := eip.GetZone()
 	if eipZone != nil {
 		insZone := self.getZone()
@@ -2931,6 +2943,7 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke
 
 	eip, err := ElasticipManager.NewEipForVMOnHost(ctx, userCred, self, host, int(bw), chargeType, eipPendingUsage)
 	if err != nil {
+		quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage)
 		return nil, httperrors.NewGeneralError(err)
 	}
 

pkg/compute/models/guest_template.go

@@ -329,6 +329,13 @@ func (gt *SGuestTemplate) getMoreDetailsV2(ctx context.Context, userCred mcclien
 		// no arrivals
 	}
 
+	// reset_password
+	if input.ResetPassword == nil {
+		configInfo.ResetPassword = false
+	} else {
+		configInfo.ResetPassword = *input.ResetPassword
+	}
+
 	out.ConfigInfo = configInfo
 	return
 }

pkg/compute/models/guestnetworks.go

@@ -540,6 +540,7 @@ func totalGuestNicCount(
 	guests := GuestManager.Query().SubQuery()
 	hosts := HostManager.Query().SubQuery()
 	guestnics := GuestnetworkManager.Query().SubQuery()
+
 	q := guestnics.Query()
 	q = q.Join(guests, sqlchemy.Equals(guests.Field("id"), guestnics.Field("guest_id")))
 	q = q.Join(hosts, sqlchemy.Equals(guests.Field("host_id"), hosts.Field("id")))

pkg/compute/models/guests.go

@@ -872,10 +872,11 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 
 func serverCreateInput2ComputeQuotaKeys(input api.ServerCreateInput, ownerId mcclient.IIdentityProvider) SComputeResourceKeys {
 	// input.Hypervisor must be set
+	brand := guessBrandForHypervisor(input.Hypervisor)
 	keys := GetDriver(input.Hypervisor).GetComputeQuotaKeys(
 		rbacutils.ScopeProject,
 		ownerId,
-		"",
+		brand,
 	)
 	if len(input.PreferHost) > 0 {
 		hostObj, _ := HostManager.FetchById(input.PreferHost)
@@ -904,40 +905,18 @@ func serverCreateInput2ComputeQuotaKeys(input api.ServerCreateInput, ownerId mcc
 func (manager *SGuestManager) BatchPreValidate(
 	ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
 	query jsonutils.JSONObject, data *jsonutils.JSONDict, count int,
-) (func(), error) {
+) error {
 	input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
 	if err != nil {
-		return nil, err
+		return errors.Wrap(err, "manager.validateCreateData")
 	}
-	if input.IsSystem == nil || *input.IsSystem == false {
-		reqQuota, reqRegionQuota, err := manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, count)
+	if input.IsSystem == nil || !(*input.IsSystem) {
+		err := manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, count)
 		if err != nil {
-			return nil, err
+			return errors.Wrap(err, "manager.checkCreateQuota")
 		}
-		quota := &SQuota{
-			Count:          reqQuota.Count / count,
-			Cpu:            reqQuota.Cpu / count,
-			Memory:         reqQuota.Memory / count,
-			Storage:        reqQuota.Storage / count,
-			IsolatedDevice: reqQuota.IsolatedDevice / count,
-		}
-		regionQuota := &SRegionQuota{
-			Port:  reqRegionQuota.Port / count,
-			Eport: reqRegionQuota.Eport / count,
-			Bw:    reqRegionQuota.Bw / count,
-			Ebw:   reqRegionQuota.Ebw / count,
-			Eip:   reqRegionQuota.Eip / count,
-		}
-		keys := serverCreateInput2ComputeQuotaKeys(*input, ownerId)
-		regionKeys := keys.SRegionalCloudResourceKeys
-		quota.SetKeys(keys)
-		regionQuota.SetKeys(regionKeys)
-		return func() {
-			quotas.CancelPendingUsage(ctx, userCred, quota, quota)
-			quotas.CancelPendingUsage(ctx, userCred, regionQuota, regionQuota)
-		}, nil
 	}
-	return nil, nil
+	return nil
 }
 
 func parseInstanceSnapshot(input *api.ServerCreateInput) (*api.ServerCreateInput, error) {
@@ -1335,7 +1314,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
 		return nil, err
 	}
 	if input.IsSystem == nil || !(*input.IsSystem) {
-		_, _, err = manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, 1)
+		err = manager.checkCreateQuota(ctx, userCred, ownerId, *input, input.Backup, 1)
 		if err != nil {
 			return nil, err
 		}
@@ -1402,17 +1381,20 @@ func (manager *SGuestManager) checkCreateQuota(
 	input api.ServerCreateInput,
 	hasBackup bool,
 	count int,
-) (*SQuota, *SRegionQuota, error) {
+) error {
 	req, regionReq := getGuestResourceRequirements(ctx, userCred, input, ownerId, count, hasBackup)
+	log.Debugf("computeQuota: %s", jsonutils.Marshal(req))
+	log.Debugf("regionQuota: %s", jsonutils.Marshal(regionReq))
+
 	err := quotas.CheckSetPendingQuota(ctx, userCred, &req)
 	if err != nil {
-		return nil, nil, err
+		return errors.Wrap(err, "quotas.CheckSetPendingQuota")
 	}
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &regionReq)
 	if err != nil {
-		return nil, nil, err
+		return errors.Wrap(err, "quotas.CheckSetPendingQuota")
 	}
-	return &req, &regionReq, nil
+	return nil
 }
 
 func (self *SGuest) checkUpdateQuota(ctx context.Context, userCred mcclient.TokenCredential, vcpuCount int, vmemSize int) (quotas.IQuota, error) {
@@ -1467,7 +1449,7 @@ func getGuestResourceRequirements(
 	eBw := 0
 	iBw := 0
 	for _, netConfig := range input.Networks {
-		if isExitNetworkInfo(netConfig) {
+		if IsExitNetworkInfo(netConfig) {
 			eNicCnt += 1
 			eBw += netConfig.BwLimit
 		} else {
@@ -1497,9 +1479,9 @@ func getGuestResourceRequirements(
 	regionReq := SRegionQuota{
 		Port:  iNicCnt * count,
 		Eport: eNicCnt * count,
-		Bw:    iBw * count,
-		Ebw:   eBw * count,
-		Eip:   eipCnt * count,
+		//Bw:    iBw * count,
+		//Ebw:   eBw * count,
+		Eip: eipCnt * count,
 	}
 	keys := serverCreateInput2ComputeQuotaKeys(input, ownerId)
 	req.SetKeys(keys)
@@ -2609,10 +2591,10 @@ func (self *SGuest) attach2NetworkOnce(
 		cancelUsage := SRegionQuota{}
 		if network.IsExitNetwork() {
 			cancelUsage.Eport = 1
-			cancelUsage.Ebw = bwLimit
+			// cancelUsage.Ebw = bwLimit
 		} else {
 			cancelUsage.Port = 1
-			cancelUsage.Bw = bwLimit
+			// cancelUsage.Bw = bwLimit
 		}
 		keys, err := self.GetRegionalQuotaKeys()
 		if err != nil {
@@ -4986,7 +4968,7 @@ func (self *SGuest) getGuestUsage(guestCount int) (SQuota, SRegionQuota, error)
 		return usage, regionUsage, err
 	}
 	regionUsage.Port = netCount
-	regionUsage.Bw = self.getBandwidth(false)
+	// regionUsage.Bw = self.getBandwidth(false)
 	eip, err := self.GetEip()
 	if err != nil && errors.Cause(err) != sql.ErrNoRows {
 		return usage, regionUsage, err

pkg/compute/models/hosts.go

@@ -1639,6 +1639,11 @@ func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.Token
 	for i := 0; i < len(removed); i += 1 {
 		log.Infof("host %s not connected with %s any more, to detach...", self.Id, removed[i].Id)
 		err := self.syncRemoveCloudHostStorage(ctx, userCred, &removed[i])
+		if errors.Cause(err) == ErrStorageInUse && removed[i].StorageType == api.STORAGE_LOCAL {
+			removed[i].SetStatus(userCred, api.STORAGE_OFFLINE, "the only host used this local storage has detached")
+			// prevent generating a delete error for syncResult
+			continue
+		}
 		if err != nil {
 			syncResult.DeleteError(err)
 		} else {

pkg/compute/models/hoststorages.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/sqlchemy"
 
@@ -31,6 +32,8 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
+const ErrStorageInUse = errors.Error("StorageInUse")
+
 type SHoststorageManager struct {
 	SHostJointsManager
 }
@@ -257,7 +260,7 @@ func (self *SHoststorage) ValidateDeleteCondition(ctx context.Context) error {
 		return httperrors.NewInternalServerError("GetGuestDiskCount fail %s", err)
 	}
 	if cnt > 0 {
-		return httperrors.NewNotEmptyError("guest on the host are using disks on this storage")
+		return errors.Wrap(ErrStorageInUse, "guest on the host are using disks on this storage")
 	}
 	return self.SHostJointsBase.ValidateDeleteCondition(ctx)
 }

pkg/compute/models/loadbalancerlisteners.go

@@ -355,7 +355,10 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user
 
 func (lblis *SLoadbalancerListener) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	lblis.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data)
-	lblis.StartLoadBalancerListenerSyncTask(ctx, userCred, data, "")
+
+	if account := lblis.GetCloudaccount(); account != nil && account.IsPublicCloud.IsTrue() {
+		lblis.StartLoadBalancerListenerSyncTask(ctx, userCred, data, "")
+	}
 }
 
 func (lblis *SLoadbalancerListener) StartLoadBalancerListenerSyncTask(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, parentTaskId string) error {

pkg/compute/models/loadbalancers.go

@@ -147,6 +147,8 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre
 		return nil, httperrors.NewBadRequestError("cannot find region info")
 	}
 
+	data.Set("cloudregion_id", jsonutils.NewString(region.Id))
+
 	input := apis.VirtualResourceCreateInput{}
 	err := data.Unmarshal(&input)
 	if err != nil {
@@ -692,7 +694,7 @@ func (self *SLoadbalancer) DeleteEip(ctx context.Context, userCred mcclient.Toke
 }
 
 func (self *SLoadbalancer) GetEip() (*SElasticip, error) {
-	return ElasticipManager.getEipForInstance(api.EIP_ASSOCIATE_TYPE_LOADBALANCER, self.Id)
+	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_LOADBALANCER, self.Id, "")
 }
 
 func (self *SLoadbalancer) SyncLoadbalancerEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extEip cloudprovider.ICloudEIP) compare.SyncResult {

pkg/compute/models/managedresource.go

@@ -550,7 +550,7 @@ func MakeCloudProviderInfoV2(region *SCloudregion, zone *SZone, provider *SCloud
 
 		info.Provider = provider.Provider
 		info.Brand = account.Brand
-		info.CloudEnv = account.getCloudEnv()
+		info.CloudEnv = account.GetCloudEnv()
 
 		if region != nil {
 			info.RegionExternalId = region.ExternalId
@@ -602,7 +602,7 @@ func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudpr
 			info.Account = account.GetName()
 			info.AccountId = account.GetId()
 			info.Brand = account.Brand
-			info.CloudEnv = account.getCloudEnv()
+			info.CloudEnv = account.GetCloudEnv()
 		}
 
 		if region != nil {

pkg/compute/models/networks.go

@@ -135,11 +135,30 @@ func (manager *SNetworkManager) AllowCreateItem(ctx context.Context, userCred mc
 	return db.IsAdminAllowCreate(userCred, manager)
 }
 
+func (self *SNetwork) GetNetworkInterfaces() ([]SNetworkInterface, error) {
+	sq := NetworkinterfacenetworkManager.Query().SubQuery()
+	q := NetworkInterfaceManager.Query()
+	q = q.Join(sq, sqlchemy.Equals(q.Field("id"), sq.Field("networkinterface_id"))).
+		Filter(sqlchemy.Equals(sq.Field("network_id"), self.Id))
+	networkinterfaces := []SNetworkInterface{}
+	err := db.FetchModelObjects(NetworkInterfaceManager, q, &networkinterfaces)
+	if err != nil {
+		return nil, err
+	}
+	return networkinterfaces, nil
+}
+
 func (self *SNetwork) ValidateDeleteCondition(ctx context.Context) error {
 	cnt, err := self.GetTotalNicCount()
 	if err != nil {
 		return httperrors.NewInternalServerError("GetTotalNicCount fail %s", err)
 	}
+	// 弹性网卡
+	nicnt, err := self.GetNetworkInterfacesCount()
+	if err != nil {
+		return httperrors.NewInternalServerError("GetNetworkInterfacesCount fail %v", err)
+	}
+	cnt -= nicnt
 	if cnt > 0 {
 		return httperrors.NewNotEmptyError("not an empty network")
 	}
@@ -922,7 +941,7 @@ func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *api.Networ
 	return nil
 }
 
-func isExitNetworkInfo(netConfig *api.NetworkConfig) bool {
+func IsExitNetworkInfo(netConfig *api.NetworkConfig) bool {
 	if len(netConfig.Network) > 0 {
 		netObj, _ := NetworkManager.FetchById(netConfig.Network)
 		net := netObj.(*SNetwork)
@@ -1243,6 +1262,8 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 		startIp = iprange.StartIp().StepUp()
 		endIp = iprange.EndIp().StepDown()
 		input.GuestIpMask = int64(prefix.MaskLen)
+		// 根据掩码得到合法的GuestIpPrefix
+		input.GuestIpPrefix = prefix.String()
 	} else {
 		startIp, err = netutils.NewIPV4Addr(input.GuestIpStart)
 		if err != nil {
@@ -1545,9 +1566,13 @@ func isOverlapNetworks(nets []SNetwork, startIp netutils.IPV4Addr, endIp netutil
 }
 
 func (self *SNetwork) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
-	if db.IsAdminAllowCreate(userCred, self.GetModelManager()) && ownerId.GetProjectId() == userCred.GetProjectId() && self.ServerType == api.NETWORK_TYPE_GUEST {
+	if db.IsDomainAllowCreate(userCred, self.GetModelManager()) && ownerId.GetProjectId() == userCred.GetProjectId() && self.ServerType == api.NETWORK_TYPE_GUEST {
 		self.IsPublic = true
-		self.PublicScope = string(rbacutils.ScopeDomain)
+		if options.Options.NonDefaultDomainProjects {
+			self.PublicScope = string(rbacutils.ScopeDomain)
+		} else {
+			self.PublicScope = string(rbacutils.ScopeSystem)
+		}
 	} else {
 		self.IsPublic = false
 		self.PublicScope = string(rbacutils.ScopeNone)
@@ -1598,6 +1623,16 @@ func (self *SNetwork) RealDelete(ctx context.Context, userCred mcclient.TokenCre
 	db.OpsLog.LogEvent(self, db.ACT_DELOCATE, self.GetShortDesc(ctx), userCred)
 	self.SetStatus(userCred, api.NETWORK_STATUS_DELETED, "real delete")
 	self.ClearSchedDescCache()
+	networkinterfaces, err := self.GetNetworkInterfaces()
+	if err != nil {
+		return errors.Wrap(err, "GetNetworkInterfaces")
+	}
+	for i := range networkinterfaces {
+		err = networkinterfaces[i].purge(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "networkinterface.purge %s(%s)", networkinterfaces[i].Name, networkinterfaces[i].Id)
+		}
+	}
 	return self.SSharableVirtualResourceBase.Delete(ctx, userCred)
 }
 

pkg/compute/models/projectquota.go

@@ -141,6 +141,15 @@ func (self *SProjectQuota) Sub(quota quotas.IQuota) {
 	self.Secgroup = nonNegative(self.Secgroup - squota.Secgroup)
 }
 
+func (self *SProjectQuota) Allocable(request quotas.IQuota) int {
+	squota := request.(*SProjectQuota)
+	cnt := -1
+	if self.Secgroup >= 0 && squota.Secgroup > 0 && (cnt < 0 || cnt > self.Secgroup/squota.Secgroup) {
+		cnt = self.Secgroup / squota.Secgroup
+	}
+	return cnt
+}
+
 func (self *SProjectQuota) Update(quota quotas.IQuota) {
 	squota := quota.(*SProjectQuota)
 	if squota.Secgroup > 0 {

pkg/compute/models/quotas.go

@@ -258,6 +258,30 @@ func (self *SQuota) Sub(quota quotas.IQuota) {
 	self.IsolatedDevice = nonNegative(self.IsolatedDevice - squota.IsolatedDevice)
 }
 
+func (self *SQuota) Allocable(request quotas.IQuota) int {
+	squota := request.(*SQuota)
+	cnt := -1
+	if self.Count >= 0 && squota.Count > 0 && (cnt < 0 || cnt > self.Count/squota.Count) {
+		cnt = self.Count / squota.Count
+	}
+	if self.Cpu >= 0 && squota.Cpu > 0 && (cnt < 0 || cnt > self.Cpu/squota.Cpu) {
+		cnt = self.Cpu / squota.Cpu
+	}
+	if self.Memory >= 0 && squota.Memory > 0 && (cnt < 0 || cnt > self.Memory/squota.Memory) {
+		cnt = self.Memory / squota.Memory
+	}
+	if self.Storage >= 0 && squota.Storage > 0 && (cnt < 0 || cnt > self.Storage/squota.Storage) {
+		cnt = self.Storage / squota.Storage
+	}
+	if self.Group >= 0 && squota.Group > 0 && (cnt < 0 || cnt > self.Group/squota.Group) {
+		cnt = self.Group / squota.Group
+	}
+	if self.IsolatedDevice >= 0 && squota.IsolatedDevice > 0 && (cnt < 0 || cnt > self.IsolatedDevice/squota.IsolatedDevice) {
+		cnt = self.IsolatedDevice / squota.IsolatedDevice
+	}
+	return cnt
+}
+
 func (self *SQuota) Update(quota quotas.IQuota) {
 	squota := quota.(*SQuota)
 	if squota.Count > 0 {
@@ -446,7 +470,7 @@ func fetchCloudQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityP
 		account := manager.GetCloudaccount()
 		keys.Provider = account.Provider
 		keys.Brand = account.Brand
-		keys.CloudEnv = account.getCloudEnv()
+		keys.CloudEnv = account.GetCloudEnv()
 		keys.AccountId = account.Id
 		keys.ManagerId = manager.Id
 	} else {
@@ -460,15 +484,19 @@ func fetchCloudQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityP
 func fetchRegionalQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, region *SCloudregion, manager *SCloudprovider) quotas.SRegionalCloudResourceKeys {
 	keys := quotas.SRegionalCloudResourceKeys{}
 	keys.SCloudResourceKeys = fetchCloudQuotaKeys(scope, ownerId, manager)
-	keys.RegionId = region.Id
+	if region != nil {
+		keys.RegionId = region.Id
+	}
 	return keys
 }
 
 func fetchZonalQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, zone *SZone, manager *SCloudprovider) quotas.SZonalCloudResourceKeys {
 	keys := quotas.SZonalCloudResourceKeys{}
 	keys.SCloudResourceKeys = fetchCloudQuotaKeys(scope, ownerId, manager)
-	keys.RegionId = zone.CloudregionId
-	keys.ZoneId = zone.Id
+	if zone != nil {
+		keys.RegionId = zone.CloudregionId
+		keys.ZoneId = zone.Id
+	}
 	return keys
 }
 

pkg/compute/models/regionquota.go

@@ -73,8 +73,8 @@ type SRegionQuota struct {
 	Eip   int `default:"-1" allow_zero:"true"`
 	Port  int `default:"-1" allow_zero:"true"`
 	Eport int `default:"-1" allow_zero:"true"`
-	Bw    int `default:"-1" allow_zero:"true"`
-	Ebw   int `default:"-1" allow_zero:"true"`
+	// Bw    int `default:"-1" allow_zero:"true"`
+	// Ebw   int `default:"-1" allow_zero:"true"`
 
 	Snapshot int `default:"-1" allow_zero:"true"`
 
@@ -125,8 +125,8 @@ func (self *SRegionQuota) FetchSystemQuota() {
 	self.Eip = defaultValue(options.Options.DefaultEipQuota)
 	self.Port = defaultValue(options.Options.DefaultPortQuota)
 	self.Eport = defaultValue(options.Options.DefaultEportQuota)
-	self.Bw = defaultValue(options.Options.DefaultBwQuota)
-	self.Ebw = defaultValue(options.Options.DefaultEbwQuota)
+	// self.Bw = defaultValue(options.Options.DefaultBwQuota)
+	// self.Ebw = defaultValue(options.Options.DefaultEbwQuota)
 	self.Snapshot = defaultValue(options.Options.DefaultSnapshotQuota)
 	self.Bucket = defaultValue(options.Options.DefaultBucketQuota)
 	self.ObjectGB = defaultValue(options.Options.DefaultObjectGBQuota)
@@ -182,8 +182,8 @@ func (self *SRegionQuota) FetchUsage(ctx context.Context) error {
 	self.Eip = eipUsage.Total()
 	self.Port = net.InternalNicCount + net.InternalVirtualNicCount + lbnic
 	self.Eport = net.ExternalNicCount + net.ExternalVirtualNicCount
-	self.Bw = net.InternalBandwidth
-	self.Ebw = net.ExternalBandwidth
+	// self.Bw = net.InternalBandwidth
+	// self.Ebw = net.ExternalBandwidth
 
 	snapshotCount, _ := TotalSnapshotCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
 	self.Snapshot = snapshotCount
@@ -211,12 +211,12 @@ func (self *SRegionQuota) ResetNegative() {
 	if self.Eport < 0 {
 		self.Eport = 0
 	}
-	if self.Bw < 0 {
-		self.Bw = 0
-	}
-	if self.Ebw < 0 {
-		self.Ebw = 0
-	}
+	// if self.Bw < 0 {
+	//	self.Bw = 0
+	// }
+	// if self.Ebw < 0 {
+	// 	self.Ebw = 0
+	// }
 	if self.Snapshot < 0 {
 		self.Snapshot = 0
 	}
@@ -250,12 +250,12 @@ func (self *SRegionQuota) IsEmpty() bool {
 	if self.Eport > 0 {
 		return false
 	}
-	if self.Bw > 0 {
-		return false
-	}
-	if self.Ebw > 0 {
-		return false
-	}
+	// if self.Bw > 0 {
+	//	return false
+	// }
+	// if self.Ebw > 0 {
+	// 	return false
+	// }
 	if self.Snapshot > 0 {
 		return false
 	}
@@ -285,8 +285,8 @@ func (self *SRegionQuota) Add(quota quotas.IQuota) {
 	self.Port = self.Port + quotas.NonNegative(squota.Port)
 	self.Eip = self.Eip + quotas.NonNegative(squota.Eip)
 	self.Eport = self.Eport + quotas.NonNegative(squota.Eport)
-	self.Bw = self.Bw + quotas.NonNegative(squota.Bw)
-	self.Ebw = self.Ebw + quotas.NonNegative(squota.Ebw)
+	// self.Bw = self.Bw + quotas.NonNegative(squota.Bw)
+	// self.Ebw = self.Ebw + quotas.NonNegative(squota.Ebw)
 	self.Snapshot = self.Snapshot + quotas.NonNegative(squota.Snapshot)
 	self.Bucket = self.Bucket + quotas.NonNegative(squota.Bucket)
 	self.ObjectGB = self.ObjectGB + quotas.NonNegative(squota.ObjectGB)
@@ -301,8 +301,8 @@ func (self *SRegionQuota) Sub(quota quotas.IQuota) {
 	self.Port = nonNegative(self.Port - squota.Port)
 	self.Eip = nonNegative(self.Eip - squota.Eip)
 	self.Eport = nonNegative(self.Eport - squota.Eport)
-	self.Bw = nonNegative(self.Bw - squota.Bw)
-	self.Ebw = nonNegative(self.Ebw - squota.Ebw)
+	// self.Bw = nonNegative(self.Bw - squota.Bw)
+	// self.Ebw = nonNegative(self.Ebw - squota.Ebw)
 	self.Snapshot = nonNegative(self.Snapshot - squota.Snapshot)
 	self.Bucket = nonNegative(self.Bucket - squota.Bucket)
 	self.ObjectGB = nonNegative(self.ObjectGB - squota.ObjectGB)
@@ -312,6 +312,48 @@ func (self *SRegionQuota) Sub(quota quotas.IQuota) {
 	self.Loadbalancer = nonNegative(self.Loadbalancer - squota.Loadbalancer)
 }
 
+func (self *SRegionQuota) Allocable(request quotas.IQuota) int {
+	squota := request.(*SRegionQuota)
+	cnt := -1
+	if self.Port >= 0 && squota.Port > 0 && (cnt < 0 || cnt > self.Port/squota.Port) {
+		cnt = self.Port / squota.Port
+	}
+	if self.Eip >= 0 && squota.Eip > 0 && (cnt < 0 || cnt > self.Eip/squota.Eip) {
+		cnt = self.Eip / squota.Eip
+	}
+	if self.Eport >= 0 && squota.Eport > 0 && (cnt < 0 || cnt > self.Eport/squota.Eport) {
+		cnt = self.Eport / squota.Eport
+	}
+	//if self.Bw >= 0 && squota.Bw > 0 && (cnt < 0 || cnt > self.Bw/squota.Bw) {
+	//	cnt = self.Bw / squota.Bw
+	//}
+	//if self.Ebw >= 0 && squota.Ebw > 0 && (cnt < 0 || cnt > self.Ebw/squota.Ebw) {
+	//	cnt = self.Ebw / squota.Ebw
+	//}
+	if self.Snapshot >= 0 && squota.Snapshot > 0 && (cnt < 0 || cnt > self.Snapshot/squota.Snapshot) {
+		cnt = self.Snapshot / squota.Snapshot
+	}
+	if self.Bucket >= 0 && squota.Bucket > 0 && (cnt < 0 || cnt > self.Bucket/squota.Bucket) {
+		cnt = self.Bucket / squota.Bucket
+	}
+	if self.ObjectGB >= 0 && squota.ObjectGB > 0 && (cnt < 0 || cnt > self.ObjectGB/squota.ObjectGB) {
+		cnt = self.ObjectGB / squota.ObjectGB
+	}
+	if self.ObjectCnt >= 0 && squota.ObjectCnt > 0 && (cnt < 0 || cnt > self.ObjectCnt/squota.ObjectCnt) {
+		cnt = self.ObjectCnt / squota.ObjectCnt
+	}
+	if self.Rds >= 0 && squota.Rds > 0 && (cnt < 0 || cnt > self.Rds/squota.Rds) {
+		cnt = self.Rds / squota.Rds
+	}
+	if self.Cache >= 0 && squota.Cache > 0 && (cnt < 0 || cnt > self.Cache/squota.Cache) {
+		cnt = self.Cache / squota.Cache
+	}
+	if self.Loadbalancer >= 0 && squota.Loadbalancer > 0 && (cnt < 0 || cnt > self.Loadbalancer/squota.Loadbalancer) {
+		cnt = self.Loadbalancer / squota.Loadbalancer
+	}
+	return cnt
+}
+
 func (self *SRegionQuota) Update(quota quotas.IQuota) {
 	squota := quota.(*SRegionQuota)
 	if squota.Port > 0 {
@@ -323,12 +365,12 @@ func (self *SRegionQuota) Update(quota quotas.IQuota) {
 	if squota.Eport > 0 {
 		self.Eport = squota.Eport
 	}
-	if squota.Bw > 0 {
-		self.Bw = squota.Bw
-	}
-	if squota.Ebw > 0 {
-		self.Ebw = squota.Ebw
-	}
+	//if squota.Bw > 0 {
+	//	self.Bw = squota.Bw
+	//}
+	//if squota.Ebw > 0 {
+	//	self.Ebw = squota.Ebw
+	//}
 	if squota.Snapshot > 0 {
 		self.Snapshot = squota.Snapshot
 	}
@@ -365,12 +407,12 @@ func (used *SRegionQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) err
 	if quotas.Exceed(used.Eport, sreq.Eport, squota.Eport) {
 		err.Add("eport", squota.Eport, used.Eport, sreq.Eport)
 	}
-	if quotas.Exceed(used.Bw, sreq.Bw, squota.Bw) {
-		err.Add("bw", squota.Bw, used.Bw, sreq.Bw)
-	}
-	if quotas.Exceed(used.Bw, sreq.Ebw, squota.Ebw) {
-		err.Add("ebw", squota.Ebw, used.Ebw, sreq.Ebw)
-	}
+	//if quotas.Exceed(used.Bw, sreq.Bw, squota.Bw) {
+	//	err.Add("bw", squota.Bw, used.Bw, sreq.Bw)
+	//}
+	//if quotas.Exceed(used.Bw, sreq.Ebw, squota.Ebw) {
+	//	err.Add("ebw", squota.Ebw, used.Ebw, sreq.Ebw)
+	//}
 	if quotas.Exceed(used.Snapshot, sreq.Snapshot, squota.Snapshot) {
 		err.Add("snapshot", squota.Snapshot, used.Snapshot, sreq.Snapshot)
 	}
@@ -404,8 +446,8 @@ func (self *SRegionQuota) ToJSON(prefix string) jsonutils.JSONObject {
 	ret.Add(jsonutils.NewInt(int64(self.Port)), keyName(prefix, "port"))
 	ret.Add(jsonutils.NewInt(int64(self.Eip)), keyName(prefix, "eip"))
 	ret.Add(jsonutils.NewInt(int64(self.Eport)), keyName(prefix, "eport"))
-	ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw"))
-	ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw"))
+	//ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw"))
+	//ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw"))
 	ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot"))
 	ret.Add(jsonutils.NewInt(int64(self.Bucket)), keyName(prefix, "bucket"))
 	ret.Add(jsonutils.NewInt(int64(self.ObjectGB)), keyName(prefix, "object_gb"))

pkg/compute/models/service_catalog.go

@@ -102,14 +102,16 @@ func (scm *SServiceCatalogManager) ValidateCreateData(ctx context.Context, userC
 	if err != nil {
 		return nil, err
 	}
-	gt := model.(*SGuestTemplate)
-	//scope := rbacutils.String2Scope(gt.PublicScope)
-	//if !gt.IsPublic || scope != rbacutils.ScopeSystem {
-	//	return nil, httperrors.NewForbiddenError("guest template must be public in scope system")
-	//}
-	if userCred.GetProjectId() != gt.ProjectId {
-		return nil, httperrors.NewForbiddenError("guest template must has same project id with the request")
-	}
+	/*
+		gt := model.(*SGuestTemplate)
+		//scope := rbacutils.String2Scope(gt.PublicScope)
+		//if !gt.IsPublic || scope != rbacutils.ScopeSystem {
+		//	return nil, httperrors.NewForbiddenError("guest template must be public in scope system")
+		//}
+		if userCred.GetProjectId() != gt.ProjectId {
+			return nil, httperrors.NewForbiddenError("guest template must has same project id with the request")
+		}
+	*/
 
 	data := input.JSON(input)
 	data.Remove("guest_template")

pkg/compute/models/skus.go

@@ -565,6 +565,7 @@ func usableFilter(q *sqlchemy.SQuery, public_cloud bool) *sqlchemy.SQuery {
 	// 过滤出network usable的sku
 	if public_cloud {
 		iconditions := NetworkUsableZoneQueries(q.Field("zone_id"), true, true)
+		iconditions = append(iconditions, sqlchemy.IsNullOrEmpty(q.Field("zone_id"))) //Azure的zone_id可能为空
 		q = q.Filter(sqlchemy.OR(iconditions...))
 	} else {
 		// 本地IDC sku 只定义到region层级, zone id 为空.因此只能按region查询
@@ -760,6 +761,11 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem
 		q = q.IsTrue("enabled")
 	}
 
+	if domian_id, _ := data.GetString("domain_id"); len(domian_id) > 0 {
+		data.Remove("domain_id")
+		q = q.In("provider", getDomainManagerProviderSubq(domian_id))
+	}
+
 	if data.Contains("zone") {
 		zoneStr, _ := data.GetString("zone")
 		_zone, err := ZoneManager.FetchByIdOrName(userCred, zoneStr)
@@ -1124,6 +1130,7 @@ func (self *SServerSku) PerformDisable(ctx context.Context, userCred mcclient.To
 
 func (self *SServerSku) syncWithCloudSku(ctx context.Context, userCred mcclient.TokenCredential, extSku SServerSku) error {
 	_, err := db.Update(self, func() error {
+		self.ZoneId = extSku.ZoneId
 		self.PrepaidStatus = extSku.PrepaidStatus
 		self.PostpaidStatus = extSku.PostpaidStatus
 		return nil
@@ -1154,7 +1161,7 @@ func (manager *SServerSkuManager) FetchSkusByRegion(regionID string) ([]SServerS
 	return skus, nil
 }
 
-func (manager *SServerSkuManager) syncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult {
+func (manager *SServerSkuManager) SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult {
 	lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 	defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
 
@@ -1281,7 +1288,11 @@ func (manager *SServerSkuManager) FetchAllAvailableSkuIdByZoneId(zoneId string)
 
 func (manager *SServerSkuManager) initializeSkuProvider() error {
 	skus := []SServerSku{}
-	q := manager.Query().In("provider", CloudproviderManager.GetPrivateProviderProvidersQuery())
+	q := manager.Query()
+	q = q.Filter(sqlchemy.OR(
+		sqlchemy.In(q.Field("provider"), CloudproviderManager.GetPrivateProviderProvidersQuery()),
+		sqlchemy.IsNullOrEmpty(q.Field("provider")),
+	))
 	err := db.FetchModelObjects(manager, q, &skus)
 	if err != nil {
 		return errors.Wrapf(err, "initializeSkuProvider.FetchModelObjects")
@@ -1289,6 +1300,8 @@ func (manager *SServerSkuManager) initializeSkuProvider() error {
 	for _, sku := range skus {
 		_, err = db.Update(&sku, func() error {
 			sku.Provider = api.CLOUD_PROVIDER_ONECLOUD
+			sku.CloudregionId = api.DEFAULT_REGION_ID
+			sku.ZoneId = ""
 			return nil
 		})
 		if err != nil {

pkg/compute/models/skus_tools.go

@@ -318,34 +318,46 @@ func SyncElasticCacheSkus(ctx context.Context, userCred mcclient.TokenCredential
 		}
 	}
 
-	meta, err := fetchSkuResourcesMeta()
+	meta, err := FetchSkuResourcesMeta()
 	if err != nil {
-		log.Errorf("SyncElasticCacheSkus.fetchSkuResourcesMeta %s", err)
+		log.Errorf("SyncElasticCacheSkus.FetchSkuResourcesMeta %s", err)
 		return
 	}
 
 	cloudregions := fetchSkuSyncCloudregions()
 	for i := range cloudregions {
 		region := &cloudregions[i]
-		meta.SetRegionFilter(region)
-		result := ElasticcacheSkuManager.syncElasticcacheSkus(ctx, userCred, region, meta)
-		notes := fmt.Sprintf("syncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result())
-		log.Infof(notes)
+
+		if region.GetDriver().IsSupportedElasticcache() {
+			meta.SetRegionFilter(region)
+			result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta)
+			notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result())
+			log.Infof(notes)
+		} else {
+			notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s not support elasticcache", region.Name)
+			log.Infof(notes)
+		}
 	}
 }
 
 // 同步Region elasticcache sku列表.
-func syncElasticCacheSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) {
-	meta, err := fetchSkuResourcesMeta()
+func SyncElasticCacheSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) error {
+	if region.GetDriver().IsSupportedElasticcache() {
+		notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s not support elasticcache", region.Name)
+		log.Infof(notes)
+		return nil
+	}
+
+	meta, err := FetchSkuResourcesMeta()
 	if err != nil {
-		log.Errorf("syncElasticCacheSkusByRegion.fetchSkuResourcesMeta %s", err)
-		return
+		return errors.Wrap(err, "SyncElasticCacheSkusByRegion.FetchSkuResourcesMeta")
 	}
 
 	meta.SetRegionFilter(region)
-	result := ElasticcacheSkuManager.syncElasticcacheSkus(ctx, userCred, region, meta)
-	notes := fmt.Sprintf("syncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result())
+	result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta)
+	notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result())
 	log.Infof(notes)
+	return nil
 }
 
 // 全量同步sku列表.
@@ -362,9 +374,9 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt
 		}
 	}
 
-	meta, err := fetchSkuResourcesMeta()
+	meta, err := FetchSkuResourcesMeta()
 	if err != nil {
-		log.Errorf("SyncServerSkus.fetchSkuResourcesMeta %s", err)
+		log.Errorf("SyncServerSkus.FetchSkuResourcesMeta %s", err)
 		return
 	}
 
@@ -372,7 +384,7 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt
 	for i := range cloudregions {
 		region := &cloudregions[i]
 		meta.SetRegionFilter(region)
-		result := ServerSkuManager.syncServerSkus(ctx, userCred, region, meta)
+		result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta)
 		notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result())
 		log.Infof(notes)
 	}
@@ -383,19 +395,19 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt
 }
 
 // 同步指定region sku列表
-func syncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) error {
-	meta, err := fetchSkuResourcesMeta()
+func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) error {
+	meta, err := FetchSkuResourcesMeta()
 	if err != nil {
-		return errors.Wrap(err, "syncServerSkusByRegion.fetchSkuResourcesMeta")
+		return errors.Wrap(err, "SyncServerSkusByRegion.FetchSkuResourcesMeta")
 	}
 
-	result := ServerSkuManager.syncServerSkus(ctx, userCred, region, meta)
-	notes := fmt.Sprintf("syncServerSkusByRegion %s result: %s", region.Name, result.Result())
+	result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta)
+	notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result())
 	log.Infof(notes)
 	return nil
 }
 
-func fetchSkuResourcesMeta() (*SSkuResourcesMeta, error) {
+func FetchSkuResourcesMeta() (*SSkuResourcesMeta, error) {
 	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
 	meta, err := modules.OfflineCloudmeta.GetSkuSourcesMeta(s)
 	if err != nil {

pkg/compute/models/snapshotpolicy.go

@@ -189,6 +189,14 @@ func (manager *SSnapshotPolicyManager) ValidateCreateData(ctx context.Context, u
 	return data, nil
 }
 
+func (manager *SSnapshotPolicyManager) OnCreateComplete(ctx context.Context, items []db.IModel,
+	userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	for i := range items {
+		sp := items[i].(*SSnapshotPolicy)
+		sp.SetStatus(userCred, api.SNAPSHOT_POLICY_READY, "create complete")
+	}
+}
+
 // ==================================================== update =========================================================
 
 func (sp *SSnapshotPolicy) AllowPerformUpdate(ctx context.Context, userCred mcclient.TokenCredential,

pkg/compute/models/snapshotpolicydisks.go

@@ -367,7 +367,7 @@ func (sd *SSnapshotPolicyDisk) DetachBySnapshotpolicy(ctx context.Context, userC
 
 // ==================================================== create =========================================================
 
-var ErrExistSD = fmt.Errorf("snapshotpolicy disk has been exist")
+const ErrExistSD = errors.Error("snapshotpolicy disk has been exist")
 
 func (self *SSnapshotPolicyDiskManager) newSnapshotpolicyDisk(ctx context.Context, userCred mcclient.TokenCredential,
 	sp *SSnapshotPolicy, disk *SDisk) (*SSnapshotPolicyDisk, error) {

pkg/compute/models/snapshots.go

@@ -278,6 +278,12 @@ func (manager *SSnapshotManager) ValidateCreateData(
 }
 
 func (self *SSnapshot) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	// use disk's ownerId instead of default ownerId
+	diskObj, err := DiskManager.FetchById(self.DiskId)
+	if err != nil {
+		return errors.Wrap(err, "DiskManager.FetchById")
+	}
+	ownerId = diskObj.(*SDisk).GetOwnerId()
 	return self.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
 }
 

pkg/compute/models/wires.go

@@ -55,8 +55,6 @@ func init() {
 			"wires",
 		),
 	}
-	WireManager.NameLength = 9
-	WireManager.NameRequireAscii = true
 	WireManager.SetVirtualObject(WireManager)
 }
 

pkg/compute/models/zonequota.go

@@ -174,6 +174,10 @@ func (self *SZoneQuota) Sub(quota quotas.IQuota) {
 	// self.Loadbalancer = nonNegative(self.Loadbalancer - squota.Loadbalancer)
 }
 
+func (self *SZoneQuota) Allocable(request quotas.IQuota) int {
+	return -1
+}
+
 func (self *SZoneQuota) Update(quota quotas.IQuota) {
 	// squota := quota.(*SZoneQuota)
 	// if squota.Loadbalancer > 0 {

pkg/compute/options/options.go

@@ -106,7 +106,7 @@ type ComputeOptions struct {
 
 	MinimalIpAddrReusedIntervalSeconds int `help:"Minimal seconds when a release IP address can be reallocate" default:"30"`
 
-	CloudSyncWorkerCount         int `help:"how many current synchronization threads" default:"2"`
+	CloudSyncWorkerCount         int `help:"how many current synchronization threads" default:"5"`
 	CloudAutoSyncIntervalSeconds int `help:"frequency to check auto sync tasks" default:"30"`
 	DefaultSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"900"`
 	MinimalSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"300"`

pkg/compute/regiondrivers/huawei.go

@@ -2178,6 +2178,14 @@ func (self *SHuaWeiRegionDriver) ValidateCreateDBInstanceAccountData(ctx context
 	if utils.IsInStringArray(instance.Engine, []string{api.DBINSTANCE_TYPE_POSTGRESQL, api.DBINSTANCE_TYPE_SQLSERVER}) {
 		return nil, httperrors.NewInputParameterError("Not support create account for huawei cloud %s instance", instance.Engine)
 	}
+	if len(input.Name) == len(input.Password) {
+		for i := range input.Name {
+			if input.Name[i] != input.Password[len(input.Password)-i-1] {
+				return input, nil
+			}
+		}
+		return nil, httperrors.NewInputParameterError("Huawei rds password cannot be in the same reverse order as the account")
+	}
 	return input, nil
 }
 

pkg/compute/service/service.go

@@ -84,6 +84,9 @@ func StartService() {
 		cron.AddJobAtIntervals("StartHostPingDetectionTask", time.Duration(opts.HostOfflineDetectionInterval)*time.Second, models.HostManager.PingDetectionTask)
 
 		cron.AddJobAtIntervalsWithStartRun("CalculateQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.QuotaManager.CalculateQuotaUsages, true)
+		cron.AddJobAtIntervalsWithStartRun("CalculateRegionQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.RegionQuotaManager.CalculateQuotaUsages, true)
+		cron.AddJobAtIntervalsWithStartRun("CalculateZoneQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.ZoneQuotaManager.CalculateQuotaUsages, true)
+		cron.AddJobAtIntervalsWithStartRun("CalculateProjectQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.ProjectQuotaManager.CalculateQuotaUsages, true)
 
 		cron.AddJobAtIntervalsWithStartRun("AutoSyncCloudaccountTask", time.Duration(opts.CloudAutoSyncIntervalSeconds)*time.Second, models.CloudaccountManager.AutoSyncCloudaccountTask, true)
 

pkg/compute/tasks/cloud_account_sync_skus_task.go

@@ -0,0 +1,104 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/utils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type CloudAccountSyncSkusTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(CloudAccountSyncSkusTask{})
+}
+
+func (self *CloudAccountSyncSkusTask) taskFailed(ctx context.Context, account *models.SCloudaccount, err error) {
+	account.SetStatus(self.UserCred, api.CLOUD_PROVIDER_SYNC_STATUS_ERROR, err.Error())
+	db.OpsLog.LogEvent(account, db.ACT_SYNC_CLOUD_SKUS, err.Error(), self.GetUserCred())
+	logclient.AddActionLogWithStartable(self, account, logclient.ACT_CLOUD_SYNC, err.Error(), self.UserCred, false)
+	self.SetStageFailed(ctx, err.Error())
+}
+
+func (self *CloudAccountSyncSkusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	account := obj.(*models.SCloudaccount)
+
+	regions := []models.SCloudregion{}
+	if regionId, _ := self.GetParams().GetString("cloudregion_id"); len(regionId) > 0 {
+		_region, err := db.FetchById(models.CloudregionManager, regionId)
+		if err != nil {
+			self.taskFailed(ctx, account, err)
+			return
+		}
+
+		region := _region.(*models.SCloudregion)
+		regions = append(regions, *region)
+	} else if providerId, _ := self.GetParams().GetString("cloudprovider_id"); len(providerId) > 0 {
+		provider, err := db.FetchById(models.CloudproviderManager, providerId)
+		if err != nil {
+			self.taskFailed(ctx, account, err)
+			return
+		}
+
+		_regions := provider.(*models.SCloudprovider).GetCloudproviderRegions()
+		for i := range _regions {
+			region := _regions[i].GetRegion()
+			regions = append(regions, *region)
+		}
+	} else {
+		providers := account.GetEnabledCloudproviders()
+		for _, provider := range providers {
+			ids := []string{}
+			_regions := provider.GetCloudproviderRegions()
+			for i := range _regions {
+				region := _regions[i].GetRegion()
+				if region != nil && !utils.IsInStringArray(region.GetId(), ids) {
+					regions = append(regions, *region)
+					ids = append(ids, region.GetId())
+				}
+			}
+		}
+	}
+
+	res, _ := self.GetParams().GetString("resource")
+	meta, err := models.FetchSkuResourcesMeta()
+	if err != nil {
+		self.taskFailed(ctx, account, err)
+		return
+	}
+
+	type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta) compare.SyncResult
+	var syncFunc SyncFunc
+	for _, region := range regions {
+		switch res {
+		case models.ServerSkuManager.Keyword():
+			syncFunc = models.ServerSkuManager.SyncServerSkus
+		case models.ElasticcacheSkuManager.Keyword():
+			syncFunc = models.ElasticcacheSkuManager.SyncElasticcacheSkus
+		case models.ElasticcacheSkuManager.Keyword():
+			syncFunc = models.DBInstanceSkuManager.SyncDBInstanceSkus
+		}
+
+		if syncFunc != nil {
+			if result := syncFunc(ctx, self.GetUserCred(), &region, meta); result.IsError() {
+				self.taskFailed(ctx, account, result.AllError())
+				return
+			} else {
+				log.Infof(result.Result())
+			}
+		}
+	}
+
+	self.SetStageComplete(ctx, nil)
+}

pkg/compute/tasks/dbinstance_account_create_task.go

@@ -73,14 +73,6 @@ func (self *DBInstanceAccountCreateTask) CreateDBInstanceAccount(ctx context.Con
 		return
 	}
 
-	input := api.SDBInstanceAccountCreateInput{}
-	self.GetParams().Unmarshal(&input)
-	if len(input.Privileges) == 0 {
-		account.SetStatus(self.UserCred, api.DBINSTANCE_USER_AVAILABLE, "")
-		self.SetStageComplete(ctx, nil)
-		return
-	}
-
 	iAccounts, err := iRds.GetIDBInstanceAccounts()
 	if err != nil {
 		msg := fmt.Sprintf("failed to found accounts from cloud dbinstance error: %v", err)
@@ -108,6 +100,16 @@ func (self *DBInstanceAccountCreateTask) CreateDBInstanceAccount(ctx context.Con
 		return
 	}
 
+	db.SetExternalId(account, self.UserCred, iAccount.GetGlobalId())
+
+	input := api.SDBInstanceAccountCreateInput{}
+	self.GetParams().Unmarshal(&input)
+	if len(input.Privileges) == 0 {
+		account.SetStatus(self.UserCred, api.DBINSTANCE_USER_AVAILABLE, "")
+		self.SetStageComplete(ctx, nil)
+		return
+	}
+
 	account.SetStatus(self.UserCred, api.DBINSTANCE_USER_GRANT_PRIVILEGE, "")
 	for _, privilege := range input.Privileges {
 		err = iAccount.GrantPrivilege(privilege.Database, privilege.Privilege)

pkg/compute/tasks/dbinstance_create_task.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -61,6 +62,35 @@ func (self *DBInstanceCreateTask) OnCreateDBInstanceComplete(ctx context.Context
 	dbinstance := obj.(*models.SDBInstance)
 	logclient.AddActionLogWithStartable(self, dbinstance, logclient.ACT_CREATE, nil, self.UserCred, true)
 
+	accounts, err := dbinstance.GetDBInstanceAccounts()
+	if err != nil {
+		log.Errorf("failed to get dbinstance %s account error: %v", dbinstance.Name, err)
+	}
+
+	if len(accounts) > 0 {
+		iRds, err := dbinstance.GetIDBInstance()
+		if err != nil {
+			log.Errorf("failed to found dbinstance %s error: %v", dbinstance.Name, err)
+		} else {
+			iAccounts, err := iRds.GetIDBInstanceAccounts()
+			if err != nil {
+				log.Errorf("failed to get accounts from cloud dbinstance %s error: %v", dbinstance.Name, err)
+			}
+			externalIds := map[string]string{}
+			for _, iAccount := range iAccounts {
+				externalIds[iAccount.GetName()] = iAccount.GetGlobalId()
+			}
+			for i := range accounts {
+				externalId, ok := externalIds[accounts[i].Name]
+				if !ok {
+					log.Errorf("failed to get dbinstance account %s from cloud dbinstance for set externalId", accounts[i].Name)
+				} else {
+					db.SetExternalId(&accounts[i], self.UserCred, externalId)
+				}
+			}
+		}
+	}
+
 	self.SetStage("OnSyncDBInstanceStatusComplete", nil)
 	dbinstance.StartDBInstanceSyncStatusTask(ctx, self.UserCred, nil, self.GetTaskId())
 }

pkg/compute/tasks/dbinstance_database_create_task.go

@@ -74,6 +74,35 @@ func (self *DBInstanceDatabaseCreateTask) CreateDBInstanceDatabase(ctx context.C
 		return
 	}
 
+	iDatabases, err := iRds.GetIDBInstanceDatabases()
+	if err != nil {
+		msg := fmt.Sprintf("failed to found databases from cloud dbinstance error: %v", err)
+		db.OpsLog.LogEvent(database, db.ACT_CREATE, msg, self.GetUserCred())
+		logclient.AddActionLogWithStartable(self, database, logclient.ACT_CREATE, msg, self.UserCred, false)
+		database.SetStatus(self.UserCred, api.DBINSTANCE_UNKNOWN, "")
+		self.SetStageComplete(ctx, nil)
+		return
+	}
+
+	var iDatabase cloudprovider.ICloudDBInstanceDatabase = nil
+	for i := range iDatabases {
+		if iDatabases[i].GetName() == database.Name {
+			iDatabase = iDatabases[i]
+			break
+		}
+	}
+
+	if iDatabase == nil {
+		msg := fmt.Sprintf("failed to found database %s from cloud dbinstance", database.Name)
+		db.OpsLog.LogEvent(database, db.ACT_CREATE, msg, self.GetUserCred())
+		logclient.AddActionLogWithStartable(self, database, logclient.ACT_CREATE, msg, self.UserCred, false)
+		database.SetStatus(self.UserCred, api.DBINSTANCE_UNKNOWN, "")
+		self.SetStageComplete(ctx, nil)
+		return
+	}
+
+	db.SetExternalId(database, self.UserCred, iDatabase.GetGlobalId())
+
 	input := api.SDBInstanceDatabaseCreateInput{}
 	self.GetParams().Unmarshal(&input)
 	if len(input.Accounts) == 0 {

pkg/compute/tasks/disk_batch_create_task.go

@@ -16,6 +16,7 @@ package tasks
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -50,6 +51,7 @@ func (self *DiskBatchCreateTask) getNeedScheduleDisks(objs []db.IStandaloneModel
 
 func (self *DiskBatchCreateTask) clearPendingUsage(ctx context.Context, disk *models.SDisk) {
 	ClearTaskPendingUsage(ctx, self)
+	ClearTaskPendingRegionUsage(ctx, self)
 }
 
 func (self *DiskBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) {
@@ -82,6 +84,25 @@ func (self *DiskBatchCreateTask) GetSchedParams() (*schedapi.ScheduleInput, erro
 	return ret, err
 }
 
+func (self *DiskBatchCreateTask) GetDisks() ([]*api.DiskConfig, error) {
+	input, err := self.GetSchedParams()
+	if err != nil {
+		return nil, err
+	}
+	return input.Disks, nil
+}
+
+func (self *DiskBatchCreateTask) GetFirstDisk() (*api.DiskConfig, error) {
+	disks, err := self.GetDisks()
+	if err != nil {
+		return nil, err
+	}
+	if len(disks) == 0 {
+		return nil, fmt.Errorf("Empty disks to schedule")
+	}
+	return disks[0], nil
+}
+
 func (self *DiskBatchCreateTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) {
 	self.SSchedTask.OnScheduleFailCallback(ctx, obj, reason)
 	disk := obj.(*models.SDisk)

pkg/compute/tasks/guest_batch_create_task.go

@@ -16,12 +16,14 @@ package tasks
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	schedapi "yunion.io/x/onecloud/pkg/apis/scheduler"
+	"yunion.io/x/onecloud/pkg/cloudcommon/cmdline"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -39,6 +41,34 @@ func init() {
 	taskman.RegisterTask(GuestBatchCreateTask{})
 }
 
+func (self *GuestBatchCreateTask) GetSchedParams() (*schedapi.ScheduleInput, error) {
+	params := self.GetParams()
+	input, err := cmdline.FetchScheduleInputByJSON(params)
+	if err != nil {
+		return nil, fmt.Errorf("Unmarsh to schedule input: %v", err)
+	}
+	return input, err
+}
+
+func (self *GuestBatchCreateTask) GetDisks() ([]*api.DiskConfig, error) {
+	input, err := self.GetSchedParams()
+	if err != nil {
+		return nil, err
+	}
+	return input.Disks, nil
+}
+
+func (self *GuestBatchCreateTask) GetFirstDisk() (*api.DiskConfig, error) {
+	disks, err := self.GetDisks()
+	if err != nil {
+		return nil, err
+	}
+	if len(disks) == 0 {
+		return nil, fmt.Errorf("Empty disks to schedule")
+	}
+	return disks[0], nil
+}
+
 func (self *GuestBatchCreateTask) GetCreateInput() (*api.ServerCreateInput, error) {
 	input := new(api.ServerCreateInput)
 	err := self.GetParams().Unmarshal(input)

pkg/compute/tasks/guest_start_task.go

@@ -137,8 +137,8 @@ func (self *GuestStartTask) OnStartComplete(ctx context.Context, obj db.IStandal
 		guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
 	} else {
 		logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_START, "", self.UserCred, true)
+		self.taskComplete(ctx, guest)
 	}
-	// self.taskComplete(ctx, guest)
 }
 
 func (self *GuestStartTask) OnGuestSyncstatusAfterStart(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {