Merge pull request #10098 from ioito/automated-cherry-pick-of-#10096-upstream-release-3.6

Automated cherry pick of #10096: fix(region): filter resource by classic vpc

cmd/climc/shell/notifyv2/config.go

@@ -111,7 +111,7 @@ func init() {
 	}
 	R(&ConfigGetTypesOptions{}, "notify-config-get-types", "Get all Config types", func(s *mcclient.ClientSession, args *ConfigGetTypesOptions) error {
 		param := jsonutils.Marshal(args)
-		result, err := modules.NotifyConfig.PerformClassAction(s, "get-types", param)
+		result, err := modules.NotifyReceiver.PerformClassAction(s, "get-types", param)
 		if err != nil {
 			return err
 		}

pkg/apigateway/handler/auth.go

@@ -309,9 +309,20 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 	if err != nil {
 		switch httperr := err.(type) {
 		case *httputils.JSONClientError:
+			if httperr.Code >= 500 {
+				return nil, err
+			}
 			if httperr.Code == 409 || httperr.Code == 429 {
 				return nil, err
 			}
+			switch httperr.Class {
+			case "UserNotFound", "WrongPassword":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrIncorrectUsernameOrPassword, "incorrect username or password")
+			case "UserLocked":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrUserLocked, "The user has been locked, please contact the administrator")
+			case "UserDisabled":
+				return nil, httperrors.NewJsonClientError(httperrors.ErrUserDisabled, "The user has been disabled, please contact the administrator")
+			}
 		}
 		return nil, httperrors.NewInvalidCredentialError("invalid credential")
 	}

pkg/apis/compute/snapshot_const.go

@@ -19,12 +19,13 @@ const (
 	SNAPSHOT_MANUAL = "manual"
 	SNAPSHOT_AUTO   = "auto"
 
-	SNAPSHOT_CREATING    = "creating"
-	SNAPSHOT_ROLLBACKING = "rollbacking"
-	SNAPSHOT_FAILED      = "create_failed"
-	SNAPSHOT_READY       = "ready"
-	SNAPSHOT_DELETING    = "deleting"
-	SNAPSHOT_UNKNOWN     = "unknown"
+	SNAPSHOT_CREATING      = "creating"
+	SNAPSHOT_ROLLBACKING   = "rollbacking"
+	SNAPSHOT_FAILED        = "create_failed"
+	SNAPSHOT_READY         = "ready"
+	SNAPSHOT_DELETE_FAILED = "delete_failed"
+	SNAPSHOT_DELETING      = "deleting"
+	SNAPSHOT_UNKNOWN       = "unknown"
 
 	SNAPSHOT_POLICY_CREATING = "creating"
 

pkg/apis/compute/vpcs_const.go

@@ -33,6 +33,8 @@ const (
 
 	DEFAULT_VPC_ID = "default"
 	NORMAL_VPC_ID  = "normal" // 没有关联VPC的安全组，统一使用normal
+
+	CLASSIC_VPC_NAME = "-"
 )
 
 type UsableResourceListInput struct {

pkg/apis/identity/usrext.go

@@ -31,11 +31,12 @@ type SUserExtended struct {
 	Email       string
 	Mobile      string
 
-	LocalId       int
-	LocalName     string
-	DomainName    string
-	DomainEnabled bool
-	IsLocal       bool
+	LocalId              int
+	LocalName            string
+	LocalFailedAuthCount int
+	DomainName           string
+	DomainEnabled        bool
+	IsLocal              bool
 	// IdpId         string
 	// IdpName       string
 }

pkg/cloudcommon/db/metadata.go

@@ -603,7 +603,7 @@ func (manager *SMetadataManager) GetAll(obj IModel, keys []string, keyPrefix str
 	ret := make(map[string]string)
 	for _, rec := range records {
 		if len(rec.Value) > 0 || strings.HasPrefix(rec.Key, USER_TAG_PREFIX) {
-			ret[rec.Key] = rec.Value
+			ret[strings.ToLower(rec.Key)] = rec.Value
 		}
 	}
 	return ret, nil

pkg/cloudcommon/notifyclient/notify.go

@@ -514,7 +514,7 @@ func notifyRobot(ctx context.Context, robot string, recipientId []string, isGrou
 	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
 	params := jsonutils.NewDict()
 	params.Set("robot", jsonutils.NewString(robot))
-	result, err := modules.NotifyConfig.PerformClassAction(s, "get-types", params)
+	result, err := modules.NotifyReceiver.PerformClassAction(s, "get-types", params)
 	if err != nil {
 		return err
 	}

pkg/cloudprovider/cloudprovider.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -163,12 +162,6 @@ type ProviderConfig struct {
 	ProxyFunc httputils.TransportProxyFunc
 }
 
-func (cp *ProviderConfig) HttpClient() *http.Client {
-	client := httputils.GetClient(true, 15*time.Second)
-	httputils.SetClientProxyFunc(client, cp.ProxyFunc)
-	return client
-}
-
 func (cp *ProviderConfig) AdaptiveTimeoutHttpClient() *http.Client {
 	client := httputils.GetAdaptiveTimeoutClient()
 	httputils.SetClientProxyFunc(client, cp.ProxyFunc)

pkg/cloudprovider/securitygroup.go

@@ -96,7 +96,6 @@ type SecurityRule struct {
 	Name       string
 	ExternalId string
 	Id         string
-	SrcPrority int
 }
 
 func (r SecurityRule) String() string {
@@ -257,7 +256,7 @@ func CompareRules(src, dest SecRuleInfo, debug bool) (common, inAdds, outAdds, i
 	}
 
 	var _compare = func(srcRules SecurityRuleSet, destRules SecurityRuleSet) (common, add, del SecurityRuleSet) {
-		i, j, destPriority, srcPrority := 0, 0, (dest.MinPriority-1+dest.MaxPriority)/2, (src.MinPriority-1+src.MaxPriority)/2
+		i, j, priority := 0, 0, (dest.MinPriority-1+dest.MaxPriority)/2
 		for i < len(srcRules) || j < len(destRules) {
 			if i < len(srcRules) && j < len(destRules) {
 				destRuleStr := destRules[j].String()
@@ -269,34 +268,28 @@ func CompareRules(src, dest SecRuleInfo, debug bool) (common, inAdds, outAdds, i
 				}
 				cmp := strings.Compare(destRuleStr, srcRuleStr)
 				if cmp == 0 {
-					destRules[j].SrcPrority = srcRules[i].Priority
 					destRules[j].Id = srcRules[i].Id
 					common = append(common, destRules[j])
-					if srcRules[i].Id != DEFAULT_SRC_RULE_ID {
-						srcPrority = srcRules[i].Priority
-					}
 					if destRules[j].ExternalId != DEFAULT_DEST_RULE_ID {
-						destPriority = destRules[j].Priority
+						priority = destRules[j].Priority
 					}
 					i++
 					j++
 				} else if cmp < 0 {
-					destRules[j].SrcPrority = srcPrority
-					srcPrority = addPriority(srcPrority, src.MinPriority, src.MaxPriority, src.IsOnlySupportAllowRules)
 					del = append(del, destRules[j])
 					j++
 				} else {
-					srcRules[i].Priority = addPriority(destPriority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+					priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+					srcRules[i].Priority = priority
 					add = append(add, srcRules[i])
 					i++
 				}
 			} else if i >= len(srcRules) {
-				destRules[j].SrcPrority = srcPrority
-				srcPrority = addPriority(srcPrority, src.MinPriority, src.MaxPriority, false)
 				del = append(del, destRules[j])
 				j++
 			} else if j >= len(destRules) {
-				srcRules[i].Priority = addPriority(destPriority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+				priority = addPriority(priority, dest.MinPriority, dest.MaxPriority, dest.IsOnlySupportAllowRules)
+				srcRules[i].Priority = priority
 				add = append(add, srcRules[i])
 				i++
 			}

pkg/compute/models/cloudaccounts.go

@@ -359,7 +359,8 @@ func (scm *SCloudaccountManager) AllowPerformPrepareNets(_ context.Context, user
 
 type sNetworkInfo struct {
 	esxi.SNetworkInfo
-	prefix string
+	prefix   string
+	fakeVsId string
 }
 
 func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, client *esxi.SESXiClient,
@@ -376,9 +377,11 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien
 		if err != nil {
 			return nil, errors.Wrap(err, "unable to fetch ips of hosts and vms")
 		}
+		nInfo.IPPool.FillVsId(caName)
 		ret = append(ret, sNetworkInfo{
 			SNetworkInfo: nInfo,
 			prefix:       caName,
+			fakeVsId:     caName,
 		})
 	case api.CLOUD_ACCOUNT_WIRE_LEVEL_DATACENTER:
 		dcs, err := client.GetDatacenters()
@@ -390,9 +393,12 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien
 			if err != nil {
 				return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q", dc.GetName())
 			}
+			prefix := fmt.Sprintf("%s/%s", caName, dc.GetName())
+			nInfo.IPPool.FillVsId(prefix)
 			ret = append(ret, sNetworkInfo{
 				SNetworkInfo: nInfo,
-				prefix:       fmt.Sprintf("%s/%s", caName, dc.GetName()),
+				prefix:       prefix,
+				fakeVsId:     prefix,
 			})
 		}
 	case api.CLOUD_ACCOUNT_WIRE_LEVEL_CLUSTER:
@@ -410,9 +416,12 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien
 				if err != nil {
 					return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q cluster %q", dc.GetName(), cluster.GetName())
 				}
+				prefix := fmt.Sprintf("%s/%s/%s", caName, dc.GetName(), cluster.GetName())
+				nInfo.IPPool.FillVsId(prefix)
 				ret = append(ret, sNetworkInfo{
 					SNetworkInfo: nInfo,
-					prefix:       fmt.Sprintf("%s/%s/%s", caName, dc.GetName(), cluster.GetName()),
+					prefix:       prefix,
+					fakeVsId:     prefix,
 				})
 			}
 		}
@@ -555,6 +564,33 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 		wires    = params.Wires
 		networks = params.Networks
 	)
+	// build global existedNetMap and IPPool
+	netNum := 0
+	for i := range networks {
+		netNum += len(networks[i])
+	}
+	existedNets := newIPPool(netNum)
+	for _, nets := range networks {
+		for i := range nets {
+			startIp, _ := netutils.NewIPV4Addr(nets[i].GuestIpStart)
+			endIp, _ := netutils.NewIPV4Addr(nets[i].GuestIpEnd)
+			existedNets.Insert(startIp, sSimpleNet{
+				Diff:   endIp - startIp,
+				Vlan:   int32(nets[i].VlanId),
+				Id:     nets[i].Id,
+				WireId: nets[i].WireId,
+			})
+
+		}
+	}
+	ipPoolLen := 0
+	for i := range nInfos {
+		ipPoolLen += nInfos[i].IPPool.Len()
+	}
+	ipPool := esxi.NewIPPool(ipPoolLen)
+	for i := range nInfos {
+		ipPool.Merge(&nInfos[i].IPPool)
+	}
 	output.CAWireNets = make([]api.CAWireNet, 0, len(nInfos))
 	for _, ni := range nInfos {
 		var (
@@ -569,11 +605,10 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 		}
 		// Find suitable wire and the network containing the Host IP in suitable wire.
 		var (
-			tmpSocre          int
-			maxScore          = len(ipHosts)
-			suitableWire      *SWire
-			suitableWireIndex = -1
-			suitableNetworks  map[netutils.IPV4Addr]*SNetwork
+			tmpSocre         int
+			maxScore         = len(ipHosts)
+			suitableWire     *SWire
+			suitableNetworks map[netutils.IPV4Addr]*SNetwork
 		)
 		for i, nets := range networks {
 			score := 0
@@ -595,7 +630,6 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 			if score > tmpSocre {
 				tmpSocre = score
 				suitableWire = &wires[i]
-				suitableWireIndex = i
 				suitableNetworks = tmpSNs
 			}
 			if tmpSocre == maxScore {
@@ -638,30 +672,12 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 		}
 
 		// Find the suitable network containing the VM IP, and if not, give the corresponding suggested network configuration in this project.
-		var allNets []SNetwork
-		if suitableWire != nil {
-			allNets = networks[suitableWireIndex]
-		}
-		type simpleNet struct {
-			Id   string
-			Vlan int32
-		}
-		existedNetMap := make(map[netutils.IPV4Addr]simpleNet, len(allNets))
-		for i := range allNets {
-			ipStart, _ := netutils.NewIPV4Addr(allNets[i].GuestIpStart)
-			ipEnd, _ := netutils.NewIPV4Addr(allNets[i].GuestIpEnd)
-			for ip := ipStart; ip <= ipEnd; ip++ {
-				existedNetMap[ip] = simpleNet{Id: allNets[i].Id, Vlan: int32(allNets[i].VlanId)}
-			}
-		}
-
 		for i := range wireNet.HostSuggestedNetworks {
 			ipStart, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpStart)
 			ipEnd, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpEnd)
-			existedNetMap[ipStart] = simpleNet{}
-			if ipEnd != ipStart {
-				existedNetMap[ipEnd] = simpleNet{}
-			}
+			existedNets.Insert(ipStart, sSimpleNet{
+				Diff: ipEnd - ipStart,
+			})
 		}
 
 		guests := make([]api.CAGuestNet, len(ni.VMs))
@@ -669,7 +685,7 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 			guests[i].Name = ni.VMs[i].Name
 			for _, ipvlan := range ni.VMs[i].IPVlans {
 				var suitableNetId string
-				sn, ok := existedNetMap[ipvlan.IP]
+				sn, ok := existedNets.Get(ipvlan.IP)
 				if ok {
 					suitableNetId = sn.Id
 				}
@@ -685,7 +701,7 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 		for vlan, ips := range ni.VlanIps {
 			for i := 0; i < len(ips); i++ {
 				ip := ips[i]
-				if _, ok := existedNetMap[ip]; ok {
+				if _, ok := existedNets.Get(ip); ok {
 					continue
 				}
 				net := ip.NetAddr(24)
@@ -694,20 +710,20 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 				// find startip
 				startIp := ip - 1
 				for ; startIp >= netLimitLow; startIp-- {
-					if _, ok := existedNetMap[startIp]; ok {
+					if _, ok := existedNets.Get(startIp); ok {
 						break
 					}
-					if _, ok := ni.IPPool.Get(startIp); ok {
+					if _, ok := ipPool.Get(startIp); ok {
 						break
 					}
 				}
 				endIp := ip + 1
 				for ; endIp <= netLimitUp; endIp++ {
-					if _, ok := existedNetMap[endIp]; ok {
+					if _, ok := existedNets.Get(endIp); ok {
 						break
 					}
-					if proc, ok := ni.IPPool.Get(endIp); ok {
-						if proc.VlanId == vlan {
+					if proc, ok := ipPool.Get(endIp); ok {
+						if proc.VlanId == vlan && proc.VSId == ni.fakeVsId {
 							// find one in ips
 							i++
 							continue
@@ -729,8 +745,9 @@ func (scm *SCloudaccountManager) parseAndSuggest(params sParseAndSuggest) api.Cl
 					},
 				})
 				// Avoid assigning already assigned ip subnet
-				existedNetMap[startIp+1] = simpleNet{}
-				existedNetMap[endIp-1] = simpleNet{}
+				existedNets.Insert(startIp+1, sSimpleNet{
+					Diff: endIp - startIp - 2,
+				})
 			}
 		}
 		output.CAWireNets = append(output.CAWireNets, wireNet)
@@ -824,6 +841,64 @@ func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr
 	return ret
 }
 
+type sIPPool struct {
+	netranges    []netutils.IPV4Addr
+	simpleNetMap map[netutils.IPV4Addr]sSimpleNet
+}
+
+func newIPPool(length ...int) *sIPPool {
+	initLen := 0
+	if len(length) > 0 {
+		initLen = length[0]
+	}
+	return &sIPPool{
+		netranges:    make([]netutils.IPV4Addr, 0, initLen),
+		simpleNetMap: make(map[netutils.IPV4Addr]sSimpleNet, initLen),
+	}
+}
+
+type sSimpleNet struct {
+	Diff   netutils.IPV4Addr
+	Id     string
+	Vlan   int32
+	WireId string
+}
+
+func (pl *sIPPool) Insert(startIp netutils.IPV4Addr, sNet sSimpleNet) {
+	// TODO:check
+	index := pl.getIndex(startIp)
+	pl.netranges = append(pl.netranges, 0)
+	pl.netranges = append(pl.netranges[:index+1], pl.netranges[index:len(pl.netranges)-1]...)
+	pl.netranges[index] = startIp
+	pl.simpleNetMap[startIp] = sNet
+}
+
+func (pl *sIPPool) getIndex(ip netutils.IPV4Addr) int {
+	index := sort.Search(len(pl.netranges), func(n int) bool {
+		return pl.netranges[n] >= ip
+	})
+	return index
+}
+
+func (pl *sIPPool) Get(ip netutils.IPV4Addr) (sSimpleNet, bool) {
+	index := pl.getIndex(ip)
+	if index > len(pl.netranges) || index < 0 {
+		return sSimpleNet{}, false
+	}
+	if index < len(pl.netranges) && pl.netranges[index] == ip {
+		return pl.simpleNetMap[ip], true
+	}
+	if index == 0 {
+		return sSimpleNet{}, false
+	}
+	startIp := pl.netranges[index-1]
+	simpleNet := pl.simpleNetMap[startIp]
+	if ip-startIp <= simpleNet.Diff {
+		return simpleNet, true
+	}
+	return sSimpleNet{}, false
+}
+
 // The suggestVMNetworks give the suggest config of network that contain the IP in 'ips' and does not intersect with the network segment described in 'excludes'.
 // The suggested network mask is 24 and the gateway is x.x.x.1.
 // The suggests network is the largest network segment that meets the above conditions.

pkg/compute/models/loadbalancers.go

@@ -303,11 +303,11 @@ func (man *SLoadbalancerManager) ValidateCreateData(
 
 	var region *SCloudregion
 	if len(input.VpcId) > 0 {
-		var vpc *SVpc
-		vpc, input.VpcResourceInput, err = ValidateVpcResourceInput(userCred, input.VpcResourceInput)
+		_vpc, err := validators.ValidateModel(userCred, VpcManager, &input.VpcId)
 		if err != nil {
-			return nil, errors.Wrap(err, "ValidateVpcResourceInput")
+			return nil, err
 		}
+		vpc := _vpc.(*SVpc)
 		region, _ = vpc.GetRegion()
 	} else if len(input.ZoneId) > 0 {
 		var zone *SZone

pkg/compute/models/networks.go

@@ -298,6 +298,16 @@ func (self *SNetwork) ValidateElbNetwork(ipAddr net.IP) (*SCloudregion, *SZone,
 	return region, zone, vpc, wire, nil
 }
 
+func (self *SNetwork) GetGuestnetworks() ([]SGuestnetwork, error) {
+	q := GuestnetworkManager.Query().Equals("network_id", self.Id)
+	gns := []SGuestnetwork{}
+	err := db.FetchModelObjects(GuestnetworkManager, q, &gns)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return gns, nil
+}
+
 func (self *SNetwork) GetGuestnicsCount() (int, error) {
 	return GuestnetworkManager.Query().Equals("network_id", self.Id).IsFalse("virtual").CountWithError()
 }
@@ -1791,6 +1801,16 @@ func (self *SNetwork) RealDelete(ctx context.Context, userCred mcclient.TokenCre
 			return errors.Wrapf(err, "reservedIps.Release %s(%d)", reservedIps[i].IpAddr, reservedIps[i].Id)
 		}
 	}
+	gns, err := self.GetGuestnetworks() // delete virtual nics
+	if err != nil {
+		return errors.Wrapf(err, "GetGuestnetworks")
+	}
+	for i := range gns {
+		err = gns[i].Delete(ctx, userCred)
+		if err != nil {
+			return errors.Wrapf(err, "delete virtual nic %s(%d)", gns[i].Ifname, gns[i].RowId)
+		}
+	}
 	if err := self.SSharableVirtualResourceBase.Delete(ctx, userCred); err != nil {
 		return err
 	}

pkg/compute/models/routetables.go

@@ -155,9 +155,9 @@ func (man *SRouteTableManager) ValidateCreateData(
 	if err != nil {
 		return input, errors.Wrap(err, "validateRoutes")
 	}
-	_, input.VpcResourceInput, err = ValidateVpcResourceInput(userCred, input.VpcResourceInput)
+	_, err = validators.ValidateModel(userCred, VpcManager, &input.VpcId)
 	if err != nil {
-		return input, errors.Wrap(err, "ValidateVpcResourceInput")
+		return input, err
 	}
 	input.StatusInfrasResourceBaseCreateInput, err = man.SStatusInfrasResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StatusInfrasResourceBaseCreateInput)
 	if err != nil {

pkg/compute/models/scaling_group.go

@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -136,7 +137,7 @@ func (sgm *SScalingGroupManager) ValidateCreateData(ctx context.Context, userCre
 	input.CloudregionId = cloudregion.GetId()
 
 	// check vpc
-	_, input.VpcResourceInput, err = ValidateVpcResourceInput(userCred, input.VpcResourceInput)
+	_, err = validators.ValidateModel(userCred, VpcManager, &input.VpcId)
 	if err != nil {
 		return input, err
 	}

pkg/compute/models/secgrouprules.go

@@ -469,8 +469,6 @@ func (self *SSecurityGroup) newFromCloudSecurityGroupRule(ctx context.Context, u
 		cidr = rule.IPNet.String()
 	}
 
-	rule.Priority = rule.SrcPrority
-
 	err := rule.ValidateRule()
 	if err != nil {
 		return nil, errors.Wrapf(err, "ValidateRule")

pkg/compute/models/vpcresource.go

@@ -16,7 +16,6 @@ package models
 
 import (
 	"context"
-	"database/sql"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -26,6 +25,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -46,19 +46,6 @@ type SVpcResourceBaseManager struct {
 	SManagedResourceBaseManager
 }
 
-func ValidateVpcResourceInput(userCred mcclient.TokenCredential, input api.VpcResourceInput) (*SVpc, api.VpcResourceInput, error) {
-	vpcObj, err := VpcManager.FetchByIdOrName(userCred, input.VpcId)
-	if err != nil {
-		if errors.Cause(err) == sql.ErrNoRows {
-			return nil, input, httperrors.NewResourceNotFoundError2(VpcManager.Keyword(), input.VpcId)
-		} else {
-			return nil, input, errors.Wrap(err, "VpcManager.FetchByIdOrName")
-		}
-	}
-	input.VpcId = vpcObj.GetId()
-	return vpcObj.(*SVpc), input, nil
-}
-
 func (self *SVpcResourceBase) GetVpc() *SVpc {
 	obj, _ := VpcManager.FetchById(self.VpcId)
 	if obj == nil {
@@ -180,11 +167,16 @@ func (manager *SVpcResourceBaseManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	var err error
 	if len(query.VpcId) > 0 {
-		vpcObj, _, err := ValidateVpcResourceInput(userCred, query.VpcResourceInput)
-		if err != nil {
-			return nil, errors.Wrap(err, "ValidateVpcResourceInput")
+		switch query.VpcId {
+		case api.CLASSIC_VPC_NAME:
+			q = q.Equals("name", api.CLASSIC_VPC_NAME)
+		default:
+			_, err := validators.ValidateModel(userCred, VpcManager, &query.VpcId)
+			if err != nil {
+				return nil, err
+			}
+			q = q.Equals("vpc_id", query.VpcId)
 		}
-		q = q.Equals("vpc_id", vpcObj.GetId())
 	}
 	subq := VpcManager.Query("id").Snapshot()
 	subq, err = manager.SCloudregionResourceBaseManager.ListItemFilter(ctx, subq, userCred, query.RegionalFilterListInput)

pkg/compute/models/vpcs.go

@@ -287,7 +287,7 @@ func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(ctx context.Context,
 	vpc.IsDefault = false
 	vpc.CloudregionId = region.Id
 	vpc.SetModelManager(manager, vpc)
-	vpc.Name = "-"
+	vpc.Name = api.CLASSIC_VPC_NAME
 	vpc.IsEmulated = true
 	vpc.SetEnabled(false)
 	vpc.Status = api.VPC_STATUS_UNAVAILABLE

pkg/compute/models/wires.go

@@ -113,11 +113,11 @@ func (manager *SWireManager) ValidateCreateData(
 		input.VpcId = api.DEFAULT_VPC_ID
 	}
 
-	var vpc *SVpc
-	vpc, input.VpcResourceInput, err = ValidateVpcResourceInput(userCred, input.VpcResourceInput)
+	_vpc, err := validators.ValidateModel(userCred, VpcManager, &input.VpcId)
 	if err != nil {
-		return input, errors.Wrap(err, "ValidateVpcResourceInput")
+		return input, err
 	}
+	vpc := _vpc.(*SVpc)
 
 	if len(vpc.ManagerId) > 0 {
 		return input, httperrors.NewNotSupportedError("Currently only kvm platform supports creating wire")

pkg/compute/options/options.go

@@ -149,7 +149,8 @@ type ComputeOptions struct {
 	SyncStorageCapacityUsedIntervalMinutes int  `help:"interval sync storage capacity used" default:"20"`
 	LockStorageFromCachedimage             bool `help:"must use storage in where selected cachedimage when creating vm"`
 
-	SyncExtDiskSnapshotIntervalMinutes int `help:"sync snapshot for external disk" default:"20"`
+	SyncExtDiskSnapshotIntervalMinutes int  `help:"sync snapshot for external disk" default:"20"`
+	AutoReconcileBackupServers         bool `help:"auto reconcile backup servers" default:"false"`
 
 	SCapabilityOptions
 	SASControllerOptions

pkg/compute/regiondrivers/secgroup_azure_test.go

@@ -141,9 +141,9 @@ func TestAzureRuleSync(t *testing.T) {
 				ruleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
 			},
 			InAdds: []cloudprovider.SecurityRule{
-				ruleWithName("", "in:allow tcp 1011", 4014),
-				ruleWithName("", "in:allow tcp 1050", 4014),
-				ruleWithName("", "in:allow tcp 1002", 4014),
+				ruleWithName("", "in:allow tcp 1050", 4010),
+				ruleWithName("", "in:allow tcp 1011", 4011),
+				ruleWithName("", "in:allow tcp 1002", 4012),
 			},
 			OutAdds: []cloudprovider.SecurityRule{},
 			InDels: []cloudprovider.SecurityRule{
@@ -178,11 +178,11 @@ func TestAzureRuleSync(t *testing.T) {
 				ruleWithName("in_allow_udp_55_4014", "in:allow udp 55", 4014),
 			},
 			InAdds: []cloudprovider.SecurityRule{
-				ruleWithName("", "in:allow icmp", 2098),
-				ruleWithName("", "in:allow udp 1055", 4015),
-				ruleWithName("", "in:allow tcp 1050", 4014),
-				ruleWithName("", "in:allow tcp 1012", 4014),
-				ruleWithName("", "in:allow tcp 1002", 4014),
+				ruleWithName("", "in:allow icmp", 2096),
+				ruleWithName("", "in:allow tcp 1050", 4010),
+				ruleWithName("", "in:allow tcp 1012", 4011),
+				ruleWithName("", "in:allow tcp 1002", 4012),
+				ruleWithName("", "in:allow udp 1055", 4013),
 			},
 			OutAdds: []cloudprovider.SecurityRule{},
 			InDels: []cloudprovider.SecurityRule{

pkg/compute/service/service.go

@@ -139,7 +139,9 @@ func StartService() {
 		cron.AddJobAtIntervalsWithStartRun("CalculateInfrasQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.InfrasQuotaManager.CalculateQuotaUsages, true)
 
 		cron.AddJobAtIntervalsWithStartRun("AutoSyncCloudaccountTask", time.Duration(opts.CloudAutoSyncIntervalSeconds)*time.Second, models.CloudaccountManager.AutoSyncCloudaccountTask, true)
-		cron.AddJobAtIntervalsWithStartRun("ReconcileBackupGuests", time.Duration(opts.ReconcileGuestBackupIntervalSeconds)*time.Second, models.GuestManager.ReconcileBackupGuests, true)
+		if opts.AutoReconcileBackupServers {
+			cron.AddJobAtIntervalsWithStartRun("ReconcileBackupGuests", time.Duration(opts.ReconcileGuestBackupIntervalSeconds)*time.Second, models.GuestManager.ReconcileBackupGuests, true)
+		}
 
 		cron.AddJobAtIntervalsWithStartRun("SyncCapacityUsedForEsxiStorage", time.Duration(opts.SyncStorageCapacityUsedIntervalMinutes)*time.Minute, models.StorageManager.SyncCapacityUsedForEsxiStorage, true)
 

pkg/compute/storagedrivers/rbd.go

@@ -220,7 +220,7 @@ func (self *SRbdStorageDriver) RequestDeleteSnapshot(ctx context.Context, snapsh
 	params.Set("snapshot_id", jsonutils.NewString(snapshot.Id))
 	_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, params, false)
 	if err != nil {
-		return errors.Wrap(err, "request create snapshot")
+		return errors.Wrap(err, "request delete snapshot")
 	}
 	return nil
 }

pkg/compute/tasks/snapshot_delete_task.go

@@ -19,10 +19,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 )
@@ -72,7 +74,12 @@ func (self *SnapshotDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneMo
 	regionDriver := snapshot.GetRegionDriver()
 
 	self.SetStage("OnRequestSnapshot", nil)
-	if err := regionDriver.RequestDeleteSnapshot(ctx, snapshot, self); err != nil {
+	err := regionDriver.RequestDeleteSnapshot(ctx, snapshot, self)
+	if err != nil {
+		if errors.Cause(err) == cloudprovider.ErrNotFound {
+			self.ScheduleRun(jsonutils.Marshal(map[string]bool{"deleted": true}))
+			return
+		}
 		self.TaskFailed(ctx, snapshot, jsonutils.NewString(err.Error()))
 	}
 }
@@ -146,9 +153,7 @@ func (self *SnapshotDeleteTask) TaskComplete(ctx context.Context, snapshot *mode
 }
 
 func (self *SnapshotDeleteTask) TaskFailed(ctx context.Context, snapshot *models.SSnapshot, reason jsonutils.JSONObject) {
-	if snapshot.Status == api.SNAPSHOT_DELETING {
-		snapshot.SetStatus(self.UserCred, api.SNAPSHOT_READY, "On SnapshotDeleteTask TaskFailed")
-	}
+	snapshot.SetStatus(self.UserCred, api.SNAPSHOT_DELETE_FAILED, reason.String())
 	db.OpsLog.LogEvent(snapshot, db.ACT_SNAPSHOT_DELETE_FAIL, reason, self.UserCred)
 	logclient.AddActionLogWithStartable(self, snapshot, logclient.ACT_DELOCATE, reason, self.UserCred, false)
 	self.SetStageFailed(ctx, reason)

pkg/esxi/handler/handlers.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -94,7 +95,11 @@ func deployHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 func deleteHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	params, _, _ := appsrv.FetchEnv(ctx, w, r)
 	diskId := params["<disk_id>"]
-	disk := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
+	disk, err := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, errors.Wrapf(err, "GetDiskById(%s)", diskId))
+		return
+	}
 	if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId == nil {
 		if disk != nil {
 			_, err := disk.Delete(ctx, nil)
@@ -171,9 +176,9 @@ func fetchHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 func diskAndDiskInfo(ctx context.Context, w http.ResponseWriter, r *http.Request) (storageman.IDisk, jsonutils.JSONObject, error) {
 	params, _, body := appsrv.FetchEnv(ctx, w, r)
 	diskId := params["<disk_id>"]
-	disk := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
-	if disk == nil {
-		return nil, nil, httperrors.NewNotFoundError("disk '%s'", diskId)
+	disk, err := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
+	if err != nil {
+		return nil, nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetDiskById(%s)", diskId))
 	}
 	diskInfo, err := body.Get("disk")
 	if err != nil {

pkg/hostman/downloader/downloadhandler.go

@@ -19,6 +19,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/options"
@@ -117,9 +119,9 @@ func diskPrecheck(
 	if storage == nil {
 		return nil, httperrors.NewNotFoundError("Storage %s not found", storageId)
 	}
-	disk := storage.GetDiskById(diskId)
-	if disk == nil {
-		return nil, httperrors.NewNotFoundError("Disk %s not found", diskId)
+	disk, err := storage.GetDiskById(diskId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "GetDiskById(%s)", diskId)
 	}
 	return disk, nil
 }

pkg/hostman/guestfs/fsdriver/windows.go

@@ -356,6 +356,8 @@ func (w *SWindowsRootFs) ChangeUserPasswd(part IDiskPartition, account, gid, pub
 	tool.CheckPath()
 	success := false
 
+	// symbol ^ is escape character is batch file.
+	password = strings.ReplaceAll(password, "^", "")
 	if rinfo != nil && version.GE(rinfo.Version, "6.1") {
 		success = w.deployPublicKeyByGuest(account, password)
 	} else {
@@ -408,8 +410,6 @@ func (w *SWindowsRootFs) deployPublicKeyByGuest(uname, passwd string) bool {
 	logPath := w.guestDebugLogPath
 	chksum := stringutils2.GetMD5Hash(passwd + logPath[(len(logPath)-10):])
 
-	// symbol ^ is escape character is batch file.
-	passwd = strings.ReplaceAll(passwd, "^", "^^")
 	chgpwdScript := strings.Join([]string{
 		w.MakeGuestDebugCmd("change password step 1"),
 		strings.Join([]string{

pkg/hostman/storageman/core.go

@@ -188,7 +188,8 @@ func (s *SStorageManager) GetStorage(storageId string) IStorage {
 
 func (s *SStorageManager) GetStorageDisk(storageId, diskId string) IDisk {
 	if storage := s.GetStorage(storageId); storage != nil {
-		return storage.GetDiskById(diskId)
+		disk, _ := storage.GetDiskById(diskId)
+		return disk
 	}
 	return nil
 }
@@ -212,7 +213,8 @@ func (s *SStorageManager) GetDiskByPath(diskPath string) IDisk {
 	}
 	storage := s.GetStorageByPath(sPath)
 	if storage != nil {
-		return storage.GetDiskById(diskId)
+		disk, _ := storage.GetDiskById(diskId)
+		return disk
 	}
 	return nil
 }

pkg/hostman/storageman/diskhandlers/diskhandler.go

@@ -21,11 +21,13 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/workmanager"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/hostman/guestman"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
@@ -140,8 +142,12 @@ func getDiskStatus(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		return
 	}
 	ret := jsonutils.NewDict()
-	disk := storage.GetDiskById(diskId)
-	if disk == nil {
+	_, err := storage.GetDiskById(diskId)
+	if err != nil {
+		if errors.Cause(err) != cloudprovider.ErrNotFound {
+			hostutils.Response(ctx, w, httperrors.NewGeneralError(errors.Wrapf(err, "GetDiskById(%s)", diskId)))
+			return
+		}
 		ret.Set("status", jsonutils.NewString(compute.DISK_NOT_EXIST))
 	} else {
 		// Note: the statuses of disk on host are either exist or not exist
@@ -220,20 +226,28 @@ func perfomrDiskActions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		hostutils.Response(ctx, w, httperrors.NewNotFoundError("Storage %s not found", storageId))
 		return
 	}
-	disk := storage.GetDiskById(diskId)
 
-	if f, ok := actionFuncs[action]; !ok {
-		hostutils.Response(ctx, w, httperrors.NewNotFoundError("Not found"))
-	} else {
-		res, err := f(ctx, storage, diskId, disk, body)
-		if err != nil {
-			hostutils.Response(ctx, w, err)
-		} else if res != nil {
-			hostutils.Response(ctx, w, res)
-		} else {
-			hostutils.ResponseOk(ctx, w)
-		}
+	disk, err := storage.GetDiskById(diskId)
+	if err != nil {
+		hostutils.Response(ctx, w, httperrors.NewGeneralError(errors.Wrapf(err, "GetDiskById(%s)", diskId)))
+		return
 	}
+
+	f, ok := actionFuncs[action]
+	if !ok {
+		hostutils.Response(ctx, w, httperrors.NewNotFoundError("Action %s Not found", action))
+		return
+	}
+	res, err := f(ctx, storage, diskId, disk, body)
+	if err != nil {
+		hostutils.Response(ctx, w, err)
+		return
+	}
+	if res != nil {
+		hostutils.Response(ctx, w, res)
+		return
+	}
+	hostutils.ResponseOk(ctx, w)
 }
 
 func diskCreate(ctx context.Context, storage storageman.IStorage, diskId string, disk storageman.IDisk, body jsonutils.JSONObject) (interface{}, error) {

pkg/hostman/storageman/storage_agent.go

@@ -54,8 +54,8 @@ func NewAgentStorage(manager *SStorageManager, agent iagent.IAgent, path string)
 	return s
 }
 
-func (as *SAgentStorage) GetDiskById(diskId string) IDisk {
-	return NewAgentDisk(as, diskId)
+func (as *SAgentStorage) GetDiskById(diskId string) (IDisk, error) {
+	return NewAgentDisk(as, diskId), nil
 }
 
 func (as *SAgentStorage) CreateDiskByDiskInfo(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
@@ -74,7 +74,10 @@ func (as *SAgentStorage) CreateDiskByDiskInfo(ctx context.Context, params interf
 	if err != nil {
 		return nil, errors.Wrap(err, "as.SLocalStorage.CreateDiskByDiskinfo")
 	}
-	disk := as.GetDiskById(createParams.DiskId)
+	disk, err := as.GetDiskById(createParams.DiskId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "GetDiskById(%s)", createParams.DiskId)
+	}
 
 	_, ds, err := as.getHostAndDatastore(ctx, hd)
 	if err != nil {

pkg/hostman/storageman/storage_base.go

@@ -104,7 +104,7 @@ type IStorage interface {
 	GetCapacity() int
 
 	// Find owner disks first, if not found, call create disk
-	GetDiskById(diskId string) IDisk
+	GetDiskById(diskId string) (IDisk, error)
 	CreateDisk(diskId string) IDisk
 	RemoveDisk(IDisk)
 

pkg/hostman/storageman/storage_local.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/pkg/util/timeutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -127,25 +128,20 @@ func (s *SLocalStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
 	return res, err
 }
 
-func (s *SLocalStorage) GetDiskById(diskId string) IDisk {
+func (s *SLocalStorage) GetDiskById(diskId string) (IDisk, error) {
 	s.DiskLock.Lock()
 	defer s.DiskLock.Unlock()
 	for i := 0; i < len(s.Disks); i++ {
 		if s.Disks[i].GetId() == diskId {
-			if s.Disks[i].Probe() == nil {
-				return s.Disks[i]
-			} else {
-				return nil
-			}
+			return s.Disks[i], s.Disks[i].Probe()
 		}
 	}
 	var disk = NewLocalDisk(s, diskId)
 	if disk.Probe() == nil {
 		s.Disks = append(s.Disks, disk)
-		return disk
-	} else {
-		return nil
+		return disk, nil
 	}
+	return nil, cloudprovider.ErrNotFound
 }
 
 func (s *SLocalStorage) CreateDisk(diskId string) IDisk {

pkg/hostman/storageman/storage_nas.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
@@ -58,25 +59,20 @@ func (s *SNasStorage) CreateDisk(diskId string) IDisk {
 	return disk
 }
 
-func (s *SNasStorage) GetDiskById(diskId string) IDisk {
+func (s *SNasStorage) GetDiskById(diskId string) (IDisk, error) {
 	s.DiskLock.Lock()
 	defer s.DiskLock.Unlock()
 	for i := 0; i < len(s.Disks); i++ {
 		if s.Disks[i].GetId() == diskId {
-			if s.Disks[i].Probe() == nil {
-				return s.Disks[i]
-			} else {
-				return nil
-			}
+			return s.Disks[i], s.Disks[i].Probe()
 		}
 	}
 	var disk = s.ins.newDisk(diskId)
 	if disk.Probe() == nil {
 		s.Disks = append(s.Disks, disk)
-		return disk
-	} else {
-		return nil
+		return disk, nil
 	}
+	return nil, cloudprovider.ErrNotFound
 }
 
 func (s *SNasStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {

pkg/hostman/storageman/storage_rbd.go

@@ -33,6 +33,7 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -595,23 +596,27 @@ func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
 	return modules.Storages.Get(hostutils.GetComputeSession(context.Background()), s.StorageName, jsonutils.Marshal(content))
 }
 
-func (s *SRbdStorage) GetDiskById(diskId string) IDisk {
+func (s *SRbdStorage) GetDiskById(diskId string) (IDisk, error) {
 	s.DiskLock.Lock()
 	defer s.DiskLock.Unlock()
 	for i := 0; i < len(s.Disks); i++ {
 		if s.Disks[i].GetId() == diskId {
-			if s.Disks[i].Probe() == nil {
-				return s.Disks[i]
+			err := s.Disks[i].Probe()
+			if err != nil {
+				if errors.Cause(err) == rbd.RbdErrorNotFound {
+					return nil, cloudprovider.ErrNotFound
+				}
+				return nil, errors.Wrapf(err, "disk.Prob")
 			}
+			return s.Disks[i], nil
 		}
 	}
 	var disk = NewRBDDisk(s, diskId)
 	if disk.Probe() == nil {
 		s.Disks = append(s.Disks, disk)
-		return disk
-	} else {
-		return nil
+		return disk, nil
 	}
+	return nil, cloudprovider.ErrNotFound
 }
 
 func (s *SRbdStorage) CreateDisk(diskId string) IDisk {

pkg/httperrors/consts.go

@@ -30,7 +30,6 @@ const (
 	ErrSpecNotFound     = errors.Error("SpecNotFoundError")
 	ErrActionNotFound   = errors.Error("ActionNotFoundError")
 	ErrTenantNotFound   = errors.Error("TenantNotFoundError")
-	ErrUserNotFound     = errors.Error("UserNotFoundError")
 
 	ErrServerStatus  = errors.Error("ServerStatusError")
 	ErrInvalidStatus = errors.ErrInvalidStatus
@@ -87,6 +86,12 @@ const (
 	ErrUnsupportedProtocol = errors.Error("UnsupportedProtocol")
 
 	ErrPolicyDefinition = errors.Error("PolicyDefinitionError")
+
+	ErrUserNotFound                = errors.Error("UserNotFound")
+	ErrUserLocked                  = errors.Error("UserLocked")
+	ErrUserDisabled                = errors.Error("UserDisabled")
+	ErrWrongPassword               = errors.Error("WrongPassword")
+	ErrIncorrectUsernameOrPassword = errors.Error("IncorrectUsernameOrPassword")
 )
 
 var (
@@ -160,6 +165,12 @@ var (
 		ErrTooManyAttempts: 429,
 		ErrTooManyRequests: 429,
 
+		ErrUserLocked:   423,
+		ErrUserDisabled: 423,
+
+		ErrWrongPassword:               401,
+		ErrIncorrectUsernameOrPassword: 401,
+
 		ErrPolicyDefinition: 409,
 	}
 )

pkg/httperrors/errors.go

@@ -207,3 +207,11 @@ func NewUnclassifiedError(msg string, params ...interface{}) *httputils.JSONClie
 func NewTooLargeEntityError(msg string, params ...interface{}) *httputils.JSONClientError {
 	return httputils.NewJsonClientError(httpErrorCode[ErrTooLarge], string(ErrTooLarge), msg, params...)
 }
+
+func NewJsonClientError(err errors.Error, msg string, params ...interface{}) *httputils.JSONClientError {
+	code, ok := httpErrorCode[err]
+	if !ok {
+		code = 500
+	}
+	return httputils.NewJsonClientError(code, string(err), msg, params...)
+}

pkg/keystone/driver/ldap/ldap.go

@@ -294,6 +294,12 @@ func (self *SLDAPDriver) Authenticate(ctx context.Context, ident mcclient.SAuthe
 	)
 	if err != nil {
 		log.Errorf("LDAP AUTH error: %s", err)
+		if errors.Cause(err) == ldaputils.ErrUserNotFound {
+			return nil, httperrors.ErrUserNotFound
+		}
+		if errors.Cause(err) == ldaputils.ErrUserBadCredential {
+			return nil, httperrors.ErrWrongPassword
+		}
 		return nil, errors.Wrap(err, "Authenticate error")
 	}
 

pkg/keystone/models/identity_provider.go

@@ -1071,11 +1071,15 @@ func (self *SIdentityProvider) SyncOrCreateUser(ctx context.Context, extId strin
 	if err == nil {
 		// update
 		log.Debugf("find user %s", extName)
-		_, err := db.Update(user, func() error {
+		newName, err := db.GenerateAlterName(user, extName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "db.GenerateAlterName %s", extName)
+		}
+		_, err = db.Update(user, func() error {
 			if syncUserInfo != nil {
 				syncUserInfo(user)
 			}
-			user.Name = extName
+			user.Name = newName
 			user.DomainId = domainId
 			if user.Deleted {
 				user.MarkUnDelete()
@@ -1099,8 +1103,13 @@ func (self *SIdentityProvider) SyncOrCreateUser(ctx context.Context, extId strin
 		} else {
 			user.Enabled = tristate.False
 		}
+		domainOwnerId := &db.SOwnerId{DomainId: domainId}
+		newName, err := db.GenerateName(UserManager, domainOwnerId, extName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "db.GenerateName %s", extName)
+		}
 		user.Id = userId
-		user.Name = extName
+		user.Name = newName
 		user.DomainId = domainId
 		err = UserManager.TableSpec().Insert(ctx, user)
 		if err != nil {

pkg/keystone/models/users.go

@@ -17,7 +17,6 @@ package models
 import (
 	"context"
 	"database/sql"
-	"fmt"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -254,6 +253,7 @@ func (manager *SUserManager) FetchUserExtended(userId, userName, domainId, domai
 		users.Field("is_system_account"),
 		localUsers.Field("id", "local_id"),
 		localUsers.Field("name", "local_name"),
+		localUsers.Field("failed_auth_count", "local_failed_auth_count"),
 		domains.Field("name", "domain_name"),
 		domains.Field("enabled", "domain_enabled"),
 		// idmappings.Field("domain_id", "idp_id"),
@@ -281,6 +281,9 @@ func (manager *SUserManager) FetchUserExtended(userId, userName, domainId, domai
 	extUser := api.SUserExtended{}
 	err := q.First(&extUser)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, httperrors.ErrUserNotFound
+		}
 		return nil, errors.Wrap(err, "query")
 	}
 
@@ -311,7 +314,7 @@ func localUserVerifyPassword(user *api.SUserExtended, passwd string) error {
 	if err == nil {
 		return nil
 	}
-	return errors.Error(fmt.Sprintf("invalid password: %v", err))
+	return httperrors.ErrWrongPassword
 }
 
 // 用户列表

pkg/keystone/tokens/auth.go

@@ -19,7 +19,6 @@ import (
 	"database/sql"
 	"time"
 
-	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
@@ -30,6 +29,7 @@ import (
 	"yunion.io/x/onecloud/pkg/keystone/driver"
 	"yunion.io/x/onecloud/pkg/keystone/models"
 	"yunion.io/x/onecloud/pkg/keystone/options"
+	o "yunion.io/x/onecloud/pkg/keystone/options"
 	"yunion.io/x/onecloud/pkg/keystone/saml"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/s3auth"
@@ -73,7 +73,7 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 	}
 	if len(ident.Password.User.Name) > 0 && len(ident.Password.User.Id) == 0 && len(ident.Password.User.Domain.Id) == 0 && len(ident.Password.User.Domain.Name) == 0 {
 		// no user domain specified, try to find user domain
-		q := models.UserManager.Query().Equals("name", ident.Password.User.Name).IsTrue("enabled")
+		q := models.UserManager.Query().Equals("name", ident.Password.User.Name)
 		usrCnt, err := q.CountWithError()
 		if err != nil {
 			return nil, errors.Wrap(err, "Query user by name")
@@ -83,7 +83,7 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 			return nil, sqlchemy.ErrDuplicateEntry
 		} else if usrCnt == 0 {
 			log.Errorf("find no user with name %s", ident.Password.User.Name)
-			return nil, sqlchemy.ErrEmptyQuery
+			return nil, httperrors.ErrUserNotFound
 		} else {
 			// userCnt == 1
 			usr := models.SUser{}
@@ -93,51 +93,48 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 				return nil, errors.Wrap(err, "Query user")
 			}
 			ident.Password.User.Domain.Id = usr.DomainId
-			idps, err := models.IdentityProviderManager.FetchIdentityProvidersByUserId(usr.Id, api.PASSWORD_PROTECTED_IDPS)
-			if err != nil {
-				return nil, errors.Wrap(err, "IdentityProviderManager.FetchIdentityProvidersByUserId")
-			}
-			log.Debugf("user %s idps: %s", ident.Password.User.Name, jsonutils.Marshal(idps))
-			if len(idps) == 0 {
-				idpId = api.DEFAULT_IDP_ID
-			} else if len(idps) == 1 {
-				idpId = idps[0].Id
-			} else {
-				log.Errorf("find %d password idps for user %s", len(idps), ident.Password.User.Name)
-				return nil, sqlchemy.ErrDuplicateEntry
-			}
-		}
-	} else {
-		usrExt, err := models.UserManager.FetchUserExtended(ident.Password.User.Id, ident.Password.User.Name,
-			ident.Password.User.Domain.Id, ident.Password.User.Domain.Name)
-		if err != nil && err != sql.ErrNoRows {
-			return nil, errors.Wrap(err, "UserManager.FetchUserExtended")
+			ident.Password.User.Id = usr.Id
 		}
+	}
 
-		if err == sql.ErrNoRows {
-			// no such user locally, query domain idp
-			domain, err := models.DomainManager.FetchDomain(ident.Password.User.Domain.Id, ident.Password.User.Domain.Name)
-			if err != nil {
-				return nil, errors.Wrap(err, "DomainManager.FetchDomain")
+	usrExt, err := models.UserManager.FetchUserExtended(ident.Password.User.Id, ident.Password.User.Name,
+		ident.Password.User.Domain.Id, ident.Password.User.Domain.Name)
+	if err != nil && errors.Cause(err) != httperrors.ErrUserNotFound {
+		return nil, errors.Wrap(err, "UserManager.FetchUserExtended")
+	}
+
+	if err != nil {
+		// no such user locally, query domain idp
+		domain, err := models.DomainManager.FetchDomain(ident.Password.User.Domain.Id, ident.Password.User.Domain.Name)
+		if err != nil {
+			return nil, errors.Wrap(err, "DomainManager.FetchDomain")
+		}
+		mapping, err := models.IdmappingManager.FetchFirstEntity(domain.Id, api.IdMappingEntityDomain)
+		if err != nil {
+			return nil, errors.Wrap(err, "IdmappingManager.FetchEntity")
+		}
+		idpId = mapping.IdpId
+	} else {
+		// check enable
+		if !usrExt.Enabled {
+			if usrExt.IsLocal && usrExt.LocalFailedAuthCount > o.Options.PasswordErrorLockCount {
+				// user locked
+				return nil, httperrors.ErrUserLocked
 			}
-			mapping, err := models.IdmappingManager.FetchFirstEntity(domain.Id, api.IdMappingEntityDomain)
-			if err != nil {
-				return nil, errors.Wrap(err, "IdmappingManager.FetchEntity")
-			}
-			idpId = mapping.IdpId
+			// user disabled
+			return nil, httperrors.ErrUserLocked
+		}
+		// user exists, query user's idp
+		idps, err := models.IdentityProviderManager.FetchIdentityProvidersByUserId(usrExt.Id, api.PASSWORD_PROTECTED_IDPS)
+		if err != nil {
+			return nil, errors.Wrap(err, "IdentityProviderManager.FetchIdentityProvidersByUserId")
+		}
+		if len(idps) == 0 {
+			idpId = api.DEFAULT_IDP_ID
+		} else if len(idps) == 1 {
+			idpId = idps[0].Id
 		} else {
-			// user exists, query user's idp
-			idps, err := models.IdentityProviderManager.FetchIdentityProvidersByUserId(usrExt.Id, api.PASSWORD_PROTECTED_IDPS)
-			if err != nil {
-				return nil, errors.Wrap(err, "IdentityProviderManager.FetchIdentityProvidersByUserId")
-			}
-			if len(idps) == 0 {
-				idpId = api.DEFAULT_IDP_ID
-			} else if len(idps) == 1 {
-				idpId = idps[0].Id
-			} else {
-				return nil, sqlchemy.ErrDuplicateEntry
-			}
+			return nil, sqlchemy.ErrDuplicateEntry
 		}
 	}
 
@@ -177,7 +174,6 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 	if idp.Status == api.IdentityDriverStatusDisconnected {
 		idp.MarkConnected(ctx, models.GetDefaultAdminCred())
 	}
-
 	return usr, nil
 }
 
@@ -429,7 +425,7 @@ func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3)
 		// auth by other methods, e.g. password , etc...
 		user, err = authUserByIdentityV3(ctx, input)
 		if err != nil {
-			return nil, errors.Wrap(err, "authUserByIdentityV3")
+			return nil, err
 		}
 	}
 

pkg/keystone/tokens/handlers.go

@@ -89,7 +89,7 @@ func authenticateTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Re
 		switch errors.Cause(err) {
 		case sqlchemy.ErrDuplicateEntry:
 			httperrors.ConflictError(ctx, w, "duplicate username")
-		case httperrors.ErrTooManyAttempts, httperrors.ErrUserNotFound:
+		case httperrors.ErrTooManyAttempts, httperrors.ErrUserNotFound, httperrors.ErrUserDisabled, httperrors.ErrUserLocked, httperrors.ErrWrongPassword:
 			httperrors.GeneralServerError(ctx, w, err)
 		default:
 			httperrors.UnauthorizedError(ctx, w, "unauthorized %s", err)

pkg/monitor/alerting/conditions/query.go

@@ -99,6 +99,9 @@ func (c *QueryCondition) filterTags(tags map[string]string, details monitor.Comm
 	if _, ok := ret["ip"]; !ok {
 		ret["ip"] = tags["host_ip"]
 	}
+	if _, ok := ret["name"]; !ok {
+		ret["name"] = tags["host"]
+	}
 	for _, tag := range []string{"brand", "platform", "hypervisor"} {
 		if val, ok := ret[tag]; ok {
 			ret["brand"] = val

pkg/multicloud/aliyun/aliyun.go

@@ -205,19 +205,18 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str
 
 	resp, err := processCommonRequest(client, req)
 	if err != nil {
-		log.Errorf("request %s error %s with params %s", apiName, err, params)
-		return nil, err
+		return nil, errors.Wrapf(err, "processCommonRequest")
 	}
 	body, err := jsonutils.Parse(resp.GetHttpContentBytes())
 	if err != nil {
-		log.Errorf("parse json fail %s", err)
-		return nil, err
+		return nil, errors.Wrapf(err, "jsonutils.Parse")
 	}
 	//{"Code":"InvalidInstanceType.ValueNotSupported","HostId":"ecs.aliyuncs.com","Message":"The specified instanceType beyond the permitted range.","RequestId":"0042EE30-0EDF-48A7-A414-56229D4AD532"}
 	//{"Code":"200","Message":"successful","PageNumber":1,"PageSize":50,"RequestId":"BB4C970C-0E23-48DC-A3B0-EB21FFC70A29","RouterTableList":{"RouterTableListType":[{"CreationTime":"2017-03-19T13:37:40Z","Description":"","ResourceGroupId":"rg-acfmwie3cqoobmi","RouteTableId":"vtb-j6c60lectdi80rk5xz43g","RouteTableName":"","RouteTableType":"System","RouterId":"vrt-j6c00qrol733dg36iq4qj","RouterType":"VRouter","VSwitchIds":{"VSwitchId":["vsw-j6c3gig5ub4fmi2veyrus"]},"VpcId":"vpc-j6c86z3sh8ufhgsxwme0q"}]},"Success":true,"TotalCount":1}
+	//{"Code":"Success","Data":{"CashCoupon":[]},"Message":"Successful!","RequestId":"87AD7E9A-3F8F-460F-9934-FFFE502325EE","Success":true}
 	if body.Contains("Code") {
 		code, _ := body.GetString("Code")
-		if len(code) > 0 && !utils.IsInStringArray(code, []string{"200"}) {
+		if len(code) > 0 && !utils.IsInStringArray(code, []string{"200", "Success"}) {
 			return nil, fmt.Errorf(body.String())
 		}
 	}

pkg/multicloud/aliyun/business.go

@@ -87,14 +87,12 @@ func (self *SAliyunClient) QueryCashCoupons() ([]SCashCoupon, error) {
 	params["EffectiveOrNot"] = "True"
 	body, err := self.businessRequest("QueryCashCoupons", params)
 	if err != nil {
-		log.Errorf("QueryCashCoupons fail %s", err)
-		return nil, err
+		return nil, errors.Wrapf(err, "QueryCashCoupons")
 	}
 	coupons := make([]SCashCoupon, 0)
 	err = body.Unmarshal(&coupons, "Data", "CashCoupon")
 	if err != nil {
-		log.Errorf("Unmarshal fail %s", err)
-		return nil, err
+		return nil, errors.Wrapf(err, "body.Unmarshal")
 	}
 	return coupons, nil
 }
@@ -104,14 +102,12 @@ func (self *SAliyunClient) QueryPrepaidCards() ([]SPrepaidCard, error) {
 	params["EffectiveOrNot"] = "True"
 	body, err := self.businessRequest("QueryPrepaidCards", params)
 	if err != nil {
-		log.Errorf("QueryPrepaidCards fail %s", err)
-		return nil, err
+		return nil, errors.Wrapf(err, "QueryPrepaidCards")
 	}
 	cards := make([]SPrepaidCard, 0)
 	err = body.Unmarshal(&cards, "Data", "PrepaidCard")
 	if err != nil {
-		log.Errorf("Unmarshal fail %s", err)
-		return nil, err
+		return nil, errors.Wrapf(err, "body.Unmarshal")
 	}
 	return cards, nil
 }
@@ -123,7 +119,6 @@ func (self *SAliyunClient) SubscribeBillToOSS(bucket string) error {
 	params["SubscribeType.1"] = "InstanceDetailForBillingPeriod"
 	body, err := self.businessRequest("SubscribeBillToOSS", params)
 	if err != nil {
-		log.Errorf("SubscribeBillToOSS fail %s", err)
 		return errors.Wrap(err, "SubscribeBillToOSS")
 	}
 	log.Debugf("%s", body)

pkg/multicloud/aliyun/provider/provider.go

@@ -282,10 +282,12 @@ func (self *SAliyunProvider) GetBalance() (float64, string, error) {
 		return 0.0, api.CLOUD_PROVIDER_HEALTH_UNKNOWN, err
 	}
 	status := api.CLOUD_PROVIDER_HEALTH_NORMAL
-	if balance.AvailableAmount <= 0 {
-		status = api.CLOUD_PROVIDER_HEALTH_ARREARS
-	} else if balance.AvailableAmount < 100 {
-		status = api.CLOUD_PROVIDER_HEALTH_INSUFFICIENT
+	if balance.CreditAmount+balance.MybankCreditAmount <= 0 {
+		if balance.AvailableAmount <= 0 {
+			status = api.CLOUD_PROVIDER_HEALTH_ARREARS
+		} else if balance.AvailableAmount < 100 {
+			status = api.CLOUD_PROVIDER_HEALTH_INSUFFICIENT
+		}
 	}
 	return balance.AvailableAmount, status, nil
 }

pkg/multicloud/aliyun/shell/business.go

@@ -33,13 +33,17 @@ func init() {
 		if err != nil {
 			return err
 		}
-		printList(result2, len(result2), 0, 0, nil)
+		if len(result2) > 0 {
+			printList(result2, len(result2), 0, 0, nil)
+		}
 
 		result3, err := cli.GetClient().QueryPrepaidCards()
 		if err != nil {
 			return err
 		}
-		printList(result3, len(result3), 0, 0, nil)
+		if len(result3) > 0 {
+			printList(result3, len(result3), 0, 0, nil)
+		}
 		return nil
 	})
 

pkg/multicloud/aws/aws.go

@@ -222,7 +222,7 @@ func (self *SAwsClient) fetchRegions() error {
 }
 
 func (client *SAwsClient) getAwsSession(regionId string) (*session.Session, error) {
-	httpClient := client.cpcfg.HttpClient()
+	httpClient := client.cpcfg.AdaptiveTimeoutHttpClient()
 	s, err := session.NewSession(&sdk.Config{
 		Region: sdk.String(regionId),
 		Credentials: credentials.NewStaticCredentials(

pkg/multicloud/ctyun/ctyun.go

@@ -86,7 +86,7 @@ type SCtyunClient struct {
 }
 
 func NewSCtyunClient(cfg *CtyunClientConfig) (*SCtyunClient, error) {
-	httpClient := cfg.cpcfg.HttpClient()
+	httpClient := cfg.cpcfg.AdaptiveTimeoutHttpClient()
 	client := &SCtyunClient{
 		CtyunClientConfig: cfg,
 		httpClient:        httpClient,

pkg/multicloud/esxi/network.go

@@ -395,6 +395,15 @@ func (p *SIPPool) Has(ip netutils.IPV4Addr) bool {
 	return ok
 }
 
+func (p *SIPPool) FillVsId(vsId string) {
+	for k, v := range p.p {
+		p.p[k] = SIPProc{
+			VlanId: v.VlanId,
+			VSId:   vsId,
+		}
+	}
+}
+
 func (p *SIPPool) Get(ip netutils.IPV4Addr) (SIPProc, bool) {
 	r, ok := p.p[ip]
 	return r, ok

pkg/multicloud/google/google.go

@@ -138,7 +138,7 @@ func NewGoogleClient(cfg *GoogleClientConfig) (*SGoogleClient, error) {
 		TokenURL: google.JWTTokenURL,
 	}
 
-	httpClient := cfg.cpcfg.HttpClient()
+	httpClient := cfg.cpcfg.AdaptiveTimeoutHttpClient()
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient)
 

pkg/multicloud/huawei/huawei.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -96,8 +97,9 @@ type SHuaweiClient struct {
 
 	isMainProject bool // whether the project is the main project in the region
 
-	ownerId   string
-	ownerName string
+	ownerId         string
+	ownerName       string
+	ownerCreateTime time.Time
 
 	iregions []cloudprovider.ICloudRegion
 	iBuckets []cloudprovider.ICloudBucket
@@ -156,7 +158,7 @@ func (self *SHuaweiClient) newRegionAPIClient(regionId string) (*client.Client,
 		return nil, err
 	}
 
-	httpClient := self.cpcfg.HttpClient()
+	httpClient := self.cpcfg.AdaptiveTimeoutHttpClient()
 	cli.SetHttpClient(httpClient)
 
 	return cli, nil
@@ -168,7 +170,7 @@ func (self *SHuaweiClient) newGeneralAPIClient() (*client.Client, error) {
 		return nil, err
 	}
 
-	httpClient := self.cpcfg.HttpClient()
+	httpClient := self.cpcfg.AdaptiveTimeoutHttpClient()
 	cli.SetHttpClient(httpClient)
 
 	return cli, nil
@@ -324,6 +326,11 @@ func (self *SHuaweiClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error)
 		if strings.ToLower(project.Name) == "mos" {
 			continue
 		}
+		// https://www.huaweicloud.com/notice/2018/20190618171312411.html
+		expiredAt, _ := timeutils.ParseTimeStr("2020-09-16 00:00:00")
+		if !self.ownerCreateTime.IsZero() && self.ownerCreateTime.After(expiredAt) && strings.ToLower(project.Name) == "cn-north-1" {
+			continue
+		}
 		s := cloudprovider.SSubAccount{
 			Name:         fmt.Sprintf("%s-%s", self.cpcfg.Name, project.Name),
 			State:        api.CLOUD_PROVIDER_CONNECTED,
@@ -418,7 +425,9 @@ func (self *SHuaweiClient) GetIStorageById(id string) (cloudprovider.ICloudStora
 
 // 总账户余额
 type SAccountBalance struct {
-	AvailableAmount float64
+	AvailableAmount  float64
+	CreditAmount     float64
+	DesignatedAmount float64
 }
 
 // 账户余额
@@ -428,8 +437,8 @@ type SBalance struct {
 	Currency         string  `json:"currency"`
 	AccountID        string  `json:"account_id"`
 	AccountType      int64   `json:"account_type"`
-	DesignatedAmount *int64  `json:"designated_amount,omitempty"`
-	CreditAmount     *int64  `json:"credit_amount,omitempty"`
+	DesignatedAmount float64 `json:"designated_amount,omitempty"`
+	CreditAmount     float64 `json:"credit_amount,omitempty"`
 	MeasureUnit      int64   `json:"measure_unit"`
 }
 
@@ -440,35 +449,33 @@ func (self *SHuaweiClient) QueryAccountBalance() (*SAccountBalance, error) {
 		return nil, err
 	}
 
-	amount := float64(0)
+	result := &SAccountBalance{}
 	for _, domain := range domains {
-		v, err := self.queryDomainBalance(domain.ID)
+		balances, err := self.queryDomainBalances(domain.ID)
 		if err != nil {
 			return nil, err
 		}
-
-		amount += v
+		for _, balance := range balances {
+			result.AvailableAmount += balance.Amount
+			result.CreditAmount += balance.CreditAmount
+			result.DesignatedAmount += balance.DesignatedAmount
+		}
 	}
 
-	return &SAccountBalance{AvailableAmount: amount}, nil
+	return result, nil
 }
 
 // https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0075213309.html
-func (self *SHuaweiClient) queryDomainBalance(domainId string) (float64, error) {
+func (self *SHuaweiClient) queryDomainBalances(domainId string) ([]SBalance, error) {
 	huawei, _ := self.newGeneralAPIClient()
 	huawei.Balances.SetDomainId(domainId)
 	balances := make([]SBalance, 0)
 	err := doListAll(huawei.Balances.List, nil, &balances)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 
-	amount := float64(0)
-	for _, balance := range balances {
-		amount += balance.Amount
-	}
-
-	return amount, nil
+	return balances, nil
 }
 
 func (self *SHuaweiClient) GetVersion() string {
@@ -540,8 +547,9 @@ func (self *SHuaweiClient) GetOwnerId() (string, error) {
 	}
 
 	type user struct {
-		DomainId string `json:"domain_id"`
-		Name     string `json:"name"`
+		DomainId   string `json:"domain_id"`
+		Name       string `json:"name"`
+		CreateTime string
 	}
 
 	ret := &user{}
@@ -550,6 +558,8 @@ func (self *SHuaweiClient) GetOwnerId() (string, error) {
 		return "", errors.Wrap(err, "SHuaweiClient.GetOwnerId.DoGet")
 	}
 	self.ownerName = ret.Name
+	// 2021-02-02 02:43:28.0
+	self.ownerCreateTime, _ = timeutils.ParseTimeStr(strings.TrimSuffix(ret.CreateTime, ".0"))
 	return ret.DomainId, nil
 }
 

pkg/multicloud/huawei/provider/provider.go

@@ -199,7 +199,7 @@ func (self *SHuaweiProvider) GetBalance() (float64, string, error) {
 		return 0.0, api.CLOUD_PROVIDER_HEALTH_UNKNOWN, err
 	}
 	status := api.CLOUD_PROVIDER_HEALTH_NORMAL
-	if balance.AvailableAmount <= 0.0 {
+	if balance.AvailableAmount <= 0.0 && balance.CreditAmount <= 0.0 {
 		status = api.CLOUD_PROVIDER_HEALTH_ARREARS
 	}
 	return balance.AvailableAmount, status, nil

pkg/multicloud/huawei/shell/region.go

@@ -53,4 +53,14 @@ func init() {
 		fmt.Println(capabilities)
 		return nil
 	})
+
+	shellutils.R(&RegionListOptions{}, "subaccount-list", "List account", func(cli *huawei.SRegion, args *RegionListOptions) error {
+		accounts, err := cli.GetClient().GetSubAccounts()
+		if err != nil {
+			return err
+		}
+		printList(accounts, 0, 0, 0, nil)
+		return nil
+	})
+
 }

pkg/multicloud/qcloud/qcloud.go

@@ -522,7 +522,7 @@ func (client *SQcloudClient) getSdkClient(regionId string) (*common.Client, erro
 	if err != nil {
 		return nil, err
 	}
-	httpClient := client.cpcfg.HttpClient()
+	httpClient := client.cpcfg.AdaptiveTimeoutHttpClient()
 	cli.WithHttpTransport(httpClient.Transport)
 	return cli, nil
 }

pkg/multicloud/ucloud/ucloud.go

@@ -101,7 +101,7 @@ type SUcloudClient struct {
 // 进行资源操作时参数account 对应数据库cloudprovider表中的account字段,由accessKey和projectID两部分组成，通过"/"分割。
 // 初次导入Subaccount时，参数account对应cloudaccounts表中的account字段，即accesskey。此时projectID为空，只能进行同步子账号（项目）、查询region列表等projectId无关的操作。
 func NewUcloudClient(cfg *UcloudClientConfig) (*SUcloudClient, error) {
-	httpClient := cfg.cpcfg.HttpClient()
+	httpClient := cfg.cpcfg.AdaptiveTimeoutHttpClient()
 	client := SUcloudClient{
 		UcloudClientConfig: cfg,
 		httpClient:         httpClient,

pkg/multicloud/zstack/zstack.go

@@ -103,7 +103,7 @@ func getSignUrl(uri string) (string, error) {
 }
 
 func NewZStackClient(cfg *ZstackClientConfig) (*SZStackClient, error) {
-	httpClient := cfg.cpcfg.HttpClient()
+	httpClient := cfg.cpcfg.AdaptiveTimeoutHttpClient()
 	cli := &SZStackClient{
 		ZstackClientConfig: cfg,
 		httpClient:         httpClient,

pkg/notify/models/config.go

@@ -160,10 +160,6 @@ func (c *SConfig) StartRepullSubcontactTask(ctx context.Context, userCred mcclie
 	return nil
 }
 
-func (cm *SConfigManager) AllowPerformGetTypes(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
-	return true
-}
-
 func (cm *SConfigManager) filterContactType(cTypes []string, robot string) []string {
 	var judge func(string) bool
 	ret := make([]string, 0, len(cTypes)/2)
@@ -189,16 +185,6 @@ func (cm *SConfigManager) filterContactType(cTypes []string, robot string) []str
 	return ret
 }
 
-func (cm *SConfigManager) PerformGetTypes(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ConfigManagerGetTypesInput) (api.ConfigManagerGetTypesOutput, error) {
-	output := api.ConfigManagerGetTypesOutput{}
-	allContactType, err := cm.allContactType()
-	if err != nil {
-		return output, err
-	}
-	output.Types = sortContactType(cm.filterContactType(allContactType, input.Robot))
-	return output, nil
-}
-
 var sortedCTypes = []string{
 	api.WEBCONSOLE, api.EMAIL, api.MOBILE, api.DINGTALK, api.FEISHU, api.WORKWX, api.DINGTALK_ROBOT, api.FEISHU_ROBOT, api.WORKWX_ROBOT,
 }
@@ -407,7 +393,7 @@ func (self *SConfigManager) InitializeData() error {
 }
 
 func (cm *SConfigManager) ResourceScope() rbacutils.TRbacScope {
-	return rbacutils.ScopeUser
+	return rbacutils.ScopeSystem
 }
 
 func (cm *SConfigManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {

pkg/notify/models/receiver.go

@@ -592,6 +592,20 @@ func (rm *SReceiverManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQue
 	return q, nil
 }
 
+func (r *SReceiverManager) AllowPerformGetTypes(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return true
+}
+
+func (cm *SReceiverManager) PerformGetTypes(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ConfigManagerGetTypesInput) (api.ConfigManagerGetTypesOutput, error) {
+	output := api.ConfigManagerGetTypesOutput{}
+	allContactType, err := ConfigManager.allContactType()
+	if err != nil {
+		return output, err
+	}
+	output.Types = sortContactType(ConfigManager.filterContactType(allContactType, input.Robot))
+	return output, nil
+}
+
 func (rm *SReceiverManager) FetchCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []interface{}, fields stringutils2.SSortedStrings, isList bool) []api.ReceiverDetails {
 	sRows := rm.SStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	dRows := rm.SDomainizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)

pkg/notify/policy/defaults.go

@@ -21,11 +21,12 @@ import (
 )
 
 const (
-	PolicyActionGet    = common_policy.PolicyActionGet
-	PolicyActionList   = common_policy.PolicyActionList
-	PolicyActionCreate = common_policy.PolicyActionCreate
-	PolicyActionUpdate = common_policy.PolicyActionUpdate
-	PolicyActionDelete = common_policy.PolicyActionDelete
+	PolicyActionPerform = common_policy.PolicyActionPerform
+	PolicyActionGet     = common_policy.PolicyActionGet
+	PolicyActionList    = common_policy.PolicyActionList
+	PolicyActionCreate  = common_policy.PolicyActionCreate
+	PolicyActionUpdate  = common_policy.PolicyActionUpdate
+	PolicyActionDelete  = common_policy.PolicyActionDelete
 )
 
 var (
@@ -36,28 +37,40 @@ var (
 			Rules: []rbacutils.SRbacRule{
 				{
 					Service:  api.SERVICE_TYPE,
-					Resource: "contacts",
+					Resource: "receivers",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "receivers",
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
 				{
 					Service:  api.SERVICE_TYPE,
-					Resource: "contacts",
+					Resource: "receivers",
 					Action:   PolicyActionCreate,
 					Result:   rbacutils.Allow,
 				},
 				{
 					Service:  api.SERVICE_TYPE,
-					Resource: "contacts",
+					Resource: "receivers",
 					Action:   PolicyActionUpdate,
 					Result:   rbacutils.Allow,
 				},
 				{
 					Service:  api.SERVICE_TYPE,
-					Resource: "contacts",
+					Resource: "receivers",
 					Action:   PolicyActionDelete,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "receivers",
+					Action:   PolicyActionPerform,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 	}