Merge pull request #8656 from lvyangyang/automated-cherry-pick-of-#8655-upstream-release-3.4

Automated cherry pick of #8655: fix:域名正则校验dnsrecord name，同步中文域名取punycode

build/notify/root/opt/yunion/share/template/title/VERIFY.email

@@ -1 +1 @@
-Yunion Verify
+{{.copyright}} Verify

go.mod

@@ -144,9 +144,9 @@ require (
 	yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
+	yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
+	yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -1137,9 +1137,11 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc h1:cgwWI6k9LsXpq6yiOuEIprsXmPyPGUa60sOZu57e8Jo=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8 h1:AkPrqO37Yi3aICys9qDjTlSdVVGh2pWQ8ET+dSJmhoI=
-yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b h1:xBAzDPvvmsOBjxAj0PLHMNLVdPNOlYFHAm7T3IZ2ehs=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/guests.go

@@ -98,6 +98,9 @@ type ServerListInput struct {
 	SrcMacCheck *bool `json:"src_mac_check"`
 
 	InstanceType []string `json:"instance_type"`
+
+	// 是否调度到宿主机上
+	WithHost *bool `json:"with_host"`
 }
 
 func (input *ServerListInput) AfterUnmarshal() {
@@ -414,3 +417,16 @@ type ServerRemoteUpdateInput struct {
 	// 是否覆盖替换所有标签
 	ReplaceTags *bool `json:"replace_tags" help:"replace all remote tags"`
 }
+
+type ServerAssociateEipInput struct {
+	// swagger:ignore
+	// Deprecated
+	Eip string `json:"eip" yunion-deprecated-by:"eip_id"`
+	// 弹性公网IP的ID
+	EipId string `json:"eip_id"`
+}
+
+type ServerDissociateEipInput struct {
+	// 是否自动释放
+	AudoDelete *bool `json:"auto_delete"`
+}

pkg/apis/compute/secgroupcache.go

@@ -18,6 +18,7 @@ import "yunion.io/x/onecloud/pkg/apis"
 
 type SecurityGroupCacheDetails struct {
 	apis.StatusStandaloneResourceDetails
+	apis.ProjectizedResourceInfo
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
@@ -25,4 +26,6 @@ type SecurityGroupCacheDetails struct {
 
 	// 虚拟私有网络名称
 	Vpc string `json:"vpc"`
+
+	ProjectId string `json:"tenant_id"`
 }

pkg/apis/compute/snapshotpolicy.go

@@ -21,10 +21,10 @@ type SnapshotPolicyDetails struct {
 
 	SSnapshotPolicy
 
-	RetentionDays  int   `json:"retention_days"`
-	RepeatWeekdays []int `json:"repeat_weekdays"`
-	TimePoints     []int `json:"time_points"`
-	IsActivated    *bool `json:"is_activated,omitempty"`
+	RetentionDays         int   `json:"retention_days"`
+	RepeatWeekdaysDisplay []int `json:"repeat_weekdays_display"`
+	TimePointsDisplay     []int `json:"time_points_display"`
+	IsActivated           *bool `json:"is_activated,omitempty"`
 
 	BindingDiskCount int `json:"binding_disk_count"`
 }

pkg/apis/compute/storage_const.go

@@ -55,7 +55,8 @@ const (
 	STORAGE_LOCAL_SSD     = "local_ssd"
 	STORAGE_LOCAL_PRO     = "local_pro"
 	STORAGE_CLOUD_BASIC   = "cloud_basic"
-	STORAGE_CLOUD_PREMIUM = "cloud_premium"
+	STORAGE_CLOUD_PREMIUM = "cloud_premium" //高性能云硬盘
+	STORAGE_CLOUD_HSSD    = "cloud_hssd"    //增强型SSD云硬盘
 
 	// huawei storage type
 	STORAGE_HUAWEI_SSD  = "SSD"  // 超高IO云硬盘

pkg/apis/identity/consts.go

@@ -202,3 +202,19 @@ var (
 		},
 	}
 )
+
+func mergeConfigOptionsFrom(opt1, opt2 map[string][]string) map[string][]string {
+	for opt, values := range opt2 {
+		ovalues, _ := opt1[opt]
+		opt1[opt] = append(ovalues, values...)
+	}
+	return opt1
+}
+
+func MergeServiceConfigOptions(opts ...map[string][]string) map[string][]string {
+	ret := make(map[string][]string)
+	for i := range opts {
+		ret = mergeConfigOptionsFrom(ret, opts[i])
+	}
+	return ret
+}

pkg/apis/monitor/commalert.go

@@ -86,6 +86,8 @@ type CommonAlertUpdateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// systemalert policy may need update through operator
+	ForceUpdate bool `json:"force_update"`
 }
 
 type CommonAlertDetails struct {

pkg/cloudcommon/db/caller.go

@@ -301,9 +301,9 @@ func FetchCustomizeColumns(
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
-		objDict := jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict)
-		objDict.Update(jsonDict)
-		retVal[i] = objDict
+		// NOTE: don't use obj update jsonDict as retval
+		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
+		retVal[i] = jsonDict
 	}
 	return retVal, nil
 }

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/informer/etcd.go

@@ -156,6 +156,10 @@ func (b *EtcdBackend) put(ctx context.Context, key, val string) error {
 	return b.PutWithLease(ctx, key, val, b.leaseTTL)
 }
 
+func (b *EtcdBackend) PutSession(ctx context.Context, key, val string) error {
+	return b.client.PutSession(ctx, key, val)
+}
+
 func (b *EtcdBackend) PutWithLease(ctx context.Context, key, val string, ttlSeconds int64) error {
 	return b.client.PutWithLease(ctx, key, val, ttlSeconds)
 }

pkg/cloudcommon/informer/etcd_client.go

@@ -99,7 +99,7 @@ func (b *EtcdBackendForClient) registerClientResource(ctx context.Context, key s
 	if err != nil {
 		return err
 	}
-	if err := b.PutWithLease(ctx, clientKey, "ok", 60); err != nil {
+	if err := b.PutSession(ctx, clientKey, "ok"); err != nil {
 		return err
 	}
 	return nil

pkg/cloudcommon/informer/informer.go

@@ -92,7 +92,7 @@ func Create(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Create(ctx, obj)
 	})
 }
@@ -101,7 +101,7 @@ func Update(ctx context.Context, obj *ModelObject, oldObj *jsonutils.JSONDict) e
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Update(ctx, obj, oldObj)
 	})
 }
@@ -110,7 +110,7 @@ func Delete(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Delete(ctx, obj)
 	})
 }

pkg/cloudcommon/informer/worker.go

@@ -15,6 +15,8 @@
 package informer
 
 import (
+	"context"
+
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -29,14 +31,35 @@ func init() {
 	informerWorkerMan = appsrv.NewWorkerManager("InformerWorkerManager", 10, 10240, false)
 }
 
-func run(f func(be IInformerBackend) error) error {
+/*type noCancel struct {
+	ctx context.Context
+}
+
+func (c noCancel) Deadline() (time.Time, bool) {
+	return time.Time{}, false
+}
+
+func (c noCancel) Done() <-chan struct{} {
+	return nil
+}
+
+func (c noCancel) Err() error {
+	return nil
+}
+
+func (c noCancel) Value(key interface{}) interface{} {
+	return c.ctx.Value(key)
+}*/
+
+func run(ctx context.Context, f func(ctx context.Context, be IInformerBackend) error) error {
 	be := GetDefaultBackend()
 	if be == nil {
 		return ErrBackendNotInit
 	}
 	wf := func() {
 		nopanic.Run(func() {
-			if err := f(be); err != nil {
+			// outside context ignored cause of run in worker
+			if err := f(context.Background(), be); err != nil {
 				log.Errorf("run informer error: %v", err)
 			}
 		})

pkg/cloudcommon/notifyclient/notify.go

@@ -20,6 +20,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -36,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
@@ -159,7 +161,16 @@ func RawNotify(recipientId []string, isGroup bool, channel npk.TNotifyChannel, p
 	rawNotify(context.Background(), recipientId, isGroup, channel, priority, event, data)
 }
 
-func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+// IntelliNotify try to create receiver nonexistent if createReceiver is set to true
+func IntelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, createReceiver)
+}
+
+const noSuchReceiver = `no such receiver whose uid is '(.*)'`
+
+var noSuchReceiverRegexp = regexp.MustCompile(noSuchReceiver)
+
+func intelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
 	log.Infof("notify %s event %s priority %s", recipientId, event, priority)
 	msg := npk.SNotifyMessage{}
 	if isGroup {
@@ -182,13 +193,46 @@ func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel
 	// log.Debugf("send notification %s %s", topic, body)
 	notifyClientWorkerMan.Run(func() {
 		s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "")
-		err := npk.Notifications.Send(s, msg)
-		if err != nil {
-			log.Errorf("unable to send notification: %v", err)
+		for {
+			err := npk.Notifications.Send(s, msg)
+			if err == nil {
+				break
+			}
+			if !createReceiver {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			jerr, ok := err.(*httputils.JSONClientError)
+			if !ok {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			if jerr.Code > 500 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			match := noSuchReceiverRegexp.FindStringSubmatch(jerr.Details)
+			if match == nil || len(match) <= 1 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			receiverId := match[1]
+			createData := jsonutils.NewDict()
+			createData.Set("uid", jsonutils.NewString(receiverId))
+			_, err = modules.NotifyReceiver.Create(s, createData)
+			if err != nil {
+				log.Errorf("try to create receiver %q, but failed: %v", receiverId, err)
+				break
+			}
+			log.Infof("create receiver %q successfully", receiverId)
 		}
 	}, nil, nil)
 }
 
+func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, false)
+}
+
 func NotifyNormal(recipientId []string, isGroup bool, event string, data jsonutils.JSONObject) {
 	notifyNormal(context.Background(), recipientId, isGroup, event, data)
 }

pkg/cloudcommon/options/manager.go

@@ -124,7 +124,10 @@ func (manager *SOptionManager) DoSync(first bool) (time.Duration, error) {
 			appsrv.SetExitFlag()
 		}
 		copyOptions(manager.options, newOpts)
-		manager.session.Upload()
+		if first {
+			// upload config for the first time ONLY
+			manager.session.Upload()
+		}
 	}
 	return manager.refreshInterval, nil
 }

pkg/compute/guestdrivers/openstack.go

@@ -232,7 +232,7 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 		}
 		defer self.attachDisks(ctx, ihost, instanceId, detachDisks)
 
-		eip, err := guest.GetEip()
+		eip, err := guest.GetElasticIp()
 		if err == nil && eip != nil {
 			ieip, err := eip.GetIEip()
 			if err != nil {

pkg/compute/guestdrivers/qcloud.go

@@ -72,6 +72,7 @@ func (self *SQcloudGuestDriver) GetStorageTypes() []string {
 		api.STORAGE_CLOUD_SSD,
 		api.STORAGE_LOCAL_BASIC,
 		api.STORAGE_LOCAL_SSD,
+		api.STORAGE_CLOUD_HSSD,
 	}
 }
 
@@ -154,6 +155,10 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 			if disk.SizeMb < 100*1024 || disk.SizeMb > 16000*1024 {
 				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", disk.Backend)
 			}
+		case api.STORAGE_CLOUD_HSSD:
+			if disk.SizeMb < 20*1024 || disk.SizeMb > 32000*1024 {
+				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", disk.Backend)
+			}
 		case api.STORAGE_LOCAL_PRO:
 			return nil, httperrors.NewInputParameterError("storage %s can not be data disk", disk.Backend)
 		}
@@ -194,6 +199,10 @@ func (self *SQcloudGuestDriver) ValidateChangeConfig(ctx context.Context, userCr
 			if newDisk.SizeMb < 100*1024 || newDisk.SizeMb > 16000*1024 {
 				return httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", newDisk.Backend)
 			}
+		case api.STORAGE_CLOUD_HSSD:
+			if newDisk.SizeMb < 20*1024 || newDisk.SizeMb > 32000*1024 {
+				return httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", newDisk.Backend)
+			}
 		case api.STORAGE_LOCAL_BASIC, api.STORAGE_LOCAL_SSD, api.STORAGE_LOCAL_PRO:
 			return httperrors.NewUnsupportOperationError("Not support create local storage disks")
 		case "": //这里Backend为空有可能会导致创建出来还是local storage,依然会出错,需要用户显式指定

pkg/compute/models/cachedimages.go

@@ -684,20 +684,6 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
-		cachedImages := CachedimageManager.Query().SubQuery()
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-
-		subq := cachedImages.Query(cachedImages.Field("id"))
-		subq = subq.Join(storagecachedImages, sqlchemy.Equals(cachedImages.Field("id"), storagecachedImages.Field("cachedimage_id")))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
-	}
-
 	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
@@ -708,74 +694,74 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByRegion(q, query.RegionalFilterListInput, "id", func() *sqlchemy.SQuery {
+	{
+		var idFilter bool
 		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
 		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
+		var storages *sqlchemy.SSubQuery
+
+		if query.Valid == nil {
+			storages = StorageManager.Query().SubQuery()
+		} else if *query.Valid {
+			idFilter = true
+			storages = StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
+		} else {
+			idFilter = true
+			stroage := StorageManager.Query()
+			storages = stroage.Filter(sqlchemy.OR(sqlchemy.NotIn(stroage.Field("status"), []string{}), sqlchemy.IsFalse(stroage.Field("enabled")))).SubQuery()
+		}
 		zones := ZoneManager.Query().SubQuery()
 
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByRegion")
-	}
 
-	q, err = managedResourceFilterByZone(q, query.ZonalFilterListInput, "id", func() *sqlchemy.SQuery {
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
-
-		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
+		if len(query.HostSchedtagId) > 0 {
+			idFilter = true
+			schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
+			if err != nil {
+				if errors.Cause(err) == sql.ErrNoRows {
+					return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
+				} else {
+					return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
+				}
+			}
+			hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
+			hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
+			subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
+			subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
+		}
 		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByZone")
+
+		subq = subq.Snapshot()
+
+		subq, err = managedResourceFilterByAccount(subq, query.ManagedResourceListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "managedResourceFilterByAccount")
+		}
+
+		subq, err = managedResourceFilterByRegion(subq, query.RegionalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByRegion")
+		}
+
+		subq, err = managedResourceFilterByZone(subq, query.ZonalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByZone")
+		}
+
+		if subq.IsAltered() {
+			idFilter = true
+		}
+
+		if idFilter {
+			q = q.In("id", subq)
+		}
 	}
 
 	if len(query.ImageType) > 0 {
-		q = q.In("image_type", query.ImageType)
-	}
-
-	if len(query.HostSchedtagId) > 0 {
-		schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
-		if err != nil {
-			if errors.Cause(err) == sql.ErrNoRows {
-				return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
-			} else {
-				return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
-			}
-		}
-		subq := StoragecachedimageManager.Query("cachedimage_id")
-		storages := StorageManager.Query("id", "storagecache_id").SubQuery()
-		hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
-		hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
-		subq = subq.Join(storages, sqlchemy.Equals(storages.Field("storagecache_id"), subq.Field("storagecache_id")))
-		subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
-		subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
-		q = q.In("id", subq.SubQuery())
-	}
-
-	if query.Valid != nil {
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
-
-		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
-		if *query.Valid {
-			q = q.In("id", subq.SubQuery())
-		} else {
-			q = q.NotIn("id", subq.SubQuery())
-		}
+		q = q.Equals("image_type", query.ImageType)
 	}
 
 	return q, nil

pkg/compute/models/cloudregions.go

@@ -118,6 +118,22 @@ func (self *SCloudregion) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudregion) GetElasticIps(managerId, eipMode string) ([]SElasticip, error) {
+	q := ElasticipManager.Query().Equals("cloudregion_id", self.Id)
+	if len(managerId) > 0 {
+		q = q.Equals("manager_id", managerId)
+	}
+	if len(eipMode) > 0 {
+		q = q.Equals("mode", eipMode)
+	}
+	eips := []SElasticip{}
+	err := db.FetchModelObjects(ElasticipManager, q, &eips)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return eips, nil
+}
+
 func (self *SCloudregion) GetZoneQuery() *sqlchemy.SQuery {
 	zones := ZoneManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {
@@ -464,6 +480,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()
 		self.Environment = cloudRegion.GetCloudEnv()
+		self.SetEnabled(true)
 
 		self.IsEmulated = cloudRegion.IsEmulated()
 

pkg/compute/models/dbinstances.go

@@ -1467,28 +1467,37 @@ func (self *SDBInstance) SetZoneInfo(ctx context.Context, userCred mcclient.Toke
 	return err
 }
 
-func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) {
+func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) error {
+	region := self.GetRegion()
+	if region == nil {
+		return fmt.Errorf("failed found region for dbinstance %s", self.Name)
+	}
+	zones, err := region.GetZones()
+	if err != nil {
+		return errors.Wrapf(err, "GetZones")
+	}
+	var setZoneId = func(input string, output *string) {
+		for _, zone := range zones {
+			if strings.HasSuffix(zone.ExternalId, input) {
+				*output = zone.Id
+				break
+			}
+		}
+		return
+	}
 	zone1 := extInstance.GetZone1Id()
 	if len(zone1) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone1)
-		if zone != nil {
-			self.Zone1 = zone.GetId()
-		}
+		setZoneId(zone1, &self.Zone1)
 	}
 	zone2 := extInstance.GetZone2Id()
 	if len(zone2) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone2)
-		if zone != nil {
-			self.Zone2 = zone.GetId()
-		}
+		setZoneId(zone2, &self.Zone2)
 	}
 	zone3 := extInstance.GetZone3Id()
 	if len(zone3) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone3)
-		if zone != nil {
-			self.Zone3 = zone.GetId()
-		}
+		setZoneId(zone3, &self.Zone3)
 	}
+	return nil
 }
 
 func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extInstance cloudprovider.ICloudDBInstance) error {
@@ -1536,7 +1545,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extInstance.GetBillingType()
-			self.ExpiredAt = extInstance.GetExpiredAt()
+			if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extInstance.IsAutoRenew()
 		}
 
@@ -1626,7 +1637,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/disks.go

@@ -1543,7 +1543,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		disk.BillingType = extDisk.GetBillingType()
-		disk.ExpiredAt = extDisk.GetExpiredAt()
+		if expired := extDisk.GetExpiredAt(); !expired.IsZero() {
+			disk.ExpiredAt = expired
+		}
 		disk.AutoRenew = extDisk.IsAutoRenew()
 	}
 
@@ -1835,6 +1837,10 @@ func (self *SDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func (self *SDisk) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	err := self.DetachAllSnapshotpolicies(ctx, userCred)
+	if err != nil {
+		log.Errorf("unable to DetachAllSnapshotpolicies: %v", err)
+	}
 	return self.SVirtualResourceBase.Delete(ctx, userCred)
 }
 
@@ -2455,6 +2461,14 @@ func (self *SDisk) UpdataSnapshotsBackingDisk(backingDiskId string) error {
 func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 
 	now := time.Now()
+	log.Infof("AutoSyncExtDiskSnapshot starts: %s", now)
+
+	week := now.Weekday()
+	if week == 0 {
+		week += 7
+	}
+	timePoint := now.Hour()
+
 	q := SnapshotPolicyDiskManager.Query().LE("next_sync_time", now)
 	spds := make([]SSnapshotPolicyDisk, 0)
 	err := db.FetchModelObjects(SnapshotPolicyDiskManager, q, &spds)
@@ -2477,25 +2491,47 @@ func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCr
 
 	for i := 0; i < len(spds); i++ {
 		spd := &spds[i]
-		disk := manager.FetchDiskById(spd.DiskId)
-
+		obj, err := manager.FetchById(spd.DiskId)
+		if errors.Cause(err) == sql.ErrNoRows || errors.Cause(err) == errors.ErrNotFound {
+			err := spd.RealDetach(ctx, userCred)
+			if err != nil {
+				log.Errorf("unable to detach s %q, d %q: %v", spd.SnapshotpolicyId, spd.DiskId, err)
+			}
+			continue
+		}
+		disk := obj.(*SDisk)
 		syncResult := disk.syncSnapshots(ctx, userCred)
 		if syncResult.IsError() {
 			db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT_FAIL, syncResult.Result(), userCred)
 			continue
 		}
-		if syncResult.AddCnt == 0 {
+		sp := spMap[spd.SnapshotpolicyId]
+		repeatWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+		timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+		if isInInts(int(week), repeatWeekdays) && isInInts(timePoint, timePoints) && syncResult.AddCnt == 0 {
+			// should add one
 			continue
 		}
 		db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT, "disk auto sync snapshot successfully", userCred)
-		_, err := db.Update(spd, func() error {
-			spd.NextSyncTime = spMap[spd.GetId()].ComputeNextSyncTime(now, spd.NextSyncTime)
+		_, err = db.Update(spd, func() error {
+			newNextSyncTime := spMap[spd.SnapshotpolicyId].ComputeNextSyncTime(now)
+			spd.NextSyncTime = newNextSyncTime
 			return nil
 		})
 		if err != nil {
 			log.Errorf("unable to update NextSyncTime for snapshotpolicydisk %q %q", spd.SnapshotpolicyId, spd.DiskId)
 		}
 	}
+	log.Infof("AutoSyncExtDiskSnapshot ends: %s", time.Now())
+}
+
+func isInInts(a int, array []int) bool {
+	for _, i := range array {
+		if i == a {
+			return true
+		}
+	}
+	return false
 }
 
 func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {

pkg/compute/models/dns_recordsets.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -119,6 +120,11 @@ func (manager *SDnsRecordSetManager) ValidateCreateData(ctx context.Context, use
 		return input, err
 	}
 	input.Name = strings.ToLower(input.Name)
+	if input.Name != "*" && input.Name != "@" {
+		if !regutils.MatchDomainName(input.Name) {
+			return input, httperrors.NewInputParameterError("invalid domain name %s", input.Name)
+		}
+	}
 	if len(input.DnsZoneId) == 0 {
 		return input, httperrors.NewMissingParameterError("dns_zone_id")
 	}

pkg/compute/models/elasticcache_instances.go

@@ -649,7 +649,9 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/elasticips.go

@@ -248,19 +248,6 @@ func (manager *SElasticipManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi
 	return q, httperrors.ErrNotFound
 }
 
-func (manager *SElasticipManager) getEipsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SElasticip, error) {
-	eips := make([]SElasticip, 0)
-	q := manager.Query().Equals("cloudregion_id", region.Id)
-	if provider != nil {
-		q = q.Equals("manager_id", provider.Id)
-	}
-	err := db.FetchModelObjects(manager, q, &eips)
-	if err != nil {
-		return nil, err
-	}
-	return eips, nil
-}
-
 func (self *SElasticip) GetRegion() *SCloudregion {
 	return CloudregionManager.FetchRegionById(self.CloudregionId)
 }
@@ -329,7 +316,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 	// remoteEips := make([]cloudprovider.ICloudEIP, 0)
 	syncResult := compare.SyncResult{}
 
-	dbEips, err := manager.getEipsByRegion(region, provider)
+	dbEips, err := region.GetElasticIps(provider.Id, api.EIP_MODE_STANDALONE_EIP)
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -478,7 +465,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.
 		factory, _ := provider.GetProviderFactory()
 		if factory != nil && factory.IsSupportPrepaidResources() {
 			self.BillingType = ext.GetBillingType()
-			self.ExpiredAt = ext.GetExpiredAt()
+			if expired := ext.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = ext.IsAutoRenew()
 		}
 
@@ -981,7 +970,7 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient.
 
 	// IMPORTANT: this serves as a guard against a guest to have multiple
 	// associated elastic_ips
-	seip, _ := server.GetEip()
+	seip, _ := server.GetEipOrPublicIp()
 	if seip != nil {
 		return nil, httperrors.NewInvalidStatusError("instance is already associated with eip")
 	}

pkg/compute/models/guest_actions.go

@@ -2607,7 +2607,7 @@ func (self *SGuest) RevokeAllSecgroups(ctx context.Context, userCred mcclient.To
 }
 
 func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) {
-	eip, _ := self.GetEip()
+	eip, _ := self.GetEipOrPublicIp()
 	if eip != nil {
 		eip.DoPendingDelete(ctx, userCred)
 	}
@@ -2649,6 +2649,11 @@ func (self *SGuest) PerformCancelDelete(ctx context.Context, userCred mcclient.T
 }
 
 func (self *SGuest) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	eip, _ := self.GetEipOrPublicIp()
+	if eip != nil {
+		eip.DoCancelPendingDelete(ctx, userCred)
+	}
+
 	for _, guestdisk := range self.GetDisks() {
 		disk := guestdisk.GetDisk()
 		disk.DoCancelPendingDelete(ctx, userCred)
@@ -2864,12 +2869,12 @@ func (self *SGuest) AllowPerformAssociateEip(ctx context.Context, userCred mccli
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "associate-eip")
 }
 
-func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerAssociateEipInput) (jsonutils.JSONObject, error) {
 	if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("cannot associate eip in status %s", self.Status)
 	}
 
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		log.Errorf("Fail to get Eip %s", err)
 		return nil, httperrors.NewGeneralError(err)
@@ -2877,7 +2882,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 	if eip != nil {
 		return nil, httperrors.NewInvalidStatusError("already associate with eip")
 	}
-	eipStr := jsonutils.GetAnyString(data, []string{"eip", "eip_id"})
+	eipStr := input.EipId
 	if len(eipStr) == 0 {
 		return nil, httperrors.NewMissingParameterError("eip_id")
 	}
@@ -2952,8 +2957,8 @@ func (self *SGuest) AllowPerformDissociateEip(ctx context.Context, userCred mccl
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "dissociate-eip")
 }
 
-func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	eip, err := self.GetEip()
+func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerDissociateEipInput) (jsonutils.JSONObject, error) {
+	eip, err := self.GetElasticIp()
 	if err != nil {
 		log.Errorf("Fail to get Eip %s", err)
 		return nil, httperrors.NewGeneralError(err)
@@ -2969,7 +2974,7 @@ func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.
 
 	self.SetStatus(userCred, api.VM_DISSOCIATE_EIP, "associate eip")
 
-	autoDelete := jsonutils.QueryBoolean(data, "auto_delete", false)
+	autoDelete := (input.AudoDelete != nil && *input.AudoDelete)
 
 	err = eip.StartEipDissociateTask(ctx, userCred, autoDelete, "")
 	if err != nil {
@@ -3797,7 +3802,7 @@ func (self *SGuest) AllowPerformStreamDisksComplete(ctx context.Context, userCre
 func (self *SGuest) PerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	for _, disk := range self.GetDisks() {
 		d := disk.GetDisk()
-		if len(d.SnapshotId) > 0 {
+		if len(d.SnapshotId) > 0 && d.GetMetadata("merge_snapshot", userCred) == "true" {
 			SnapshotManager.AddRefCount(d.SnapshotId, -1)
 			d.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONFalse, userCred)
 		}
@@ -4313,7 +4318,7 @@ func (guest *SGuest) PerformChangeOwner(ctx context.Context, userCred mcclient.T
 		}
 	}
 
-	if eip, _ := guest.GetEip(); eip != nil {
+	if eip, _ := guest.GetEipOrPublicIp(); eip != nil {
 		_, err := eip.PerformChangeOwner(ctx, userCred, query, input)
 		if err != nil {
 			return nil, err

pkg/compute/models/guests.go

@@ -494,6 +494,13 @@ func (manager *SGuestManager) ListItemFilter(
 	if len(query.InstanceType) > 0 {
 		q = q.In("instance_type", query.InstanceType)
 	}
+	if query.WithHost != nil {
+		if *query.WithHost {
+			q = q.IsNotEmpty("host_id")
+		} else {
+			q = q.IsNullOrEmpty("host_id")
+		}
+	}
 
 	return q, nil
 }
@@ -2586,7 +2593,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		guest.BillingType = extVM.GetBillingType()
-		guest.ExpiredAt = extVM.GetExpiredAt()
+		if expired := extVM.GetExpiredAt(); !expired.IsZero() {
+			guest.ExpiredAt = expired
+		}
 		if guest.GetDriver().IsSupportSetAutoRenew() {
 			guest.AutoRenew = extVM.IsAutoRenew()
 		}
@@ -3121,12 +3130,11 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred
 
 func filterGuestByRange(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery {
 	hosts := HostManager.Query().SubQuery()
+	subq := hosts.Query(hosts.Field("id"))
+	subq = AttachUsageQuery(subq, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 
-	q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), q.Field("host_id")))
-	//q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-	// q = q.Filter(sqlchemy.Equals(hosts.Field("host_status"), HOST_ONLINE))
+	q = q.In("host_id", subq.SubQuery())
 
-	q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 	return q
 }
 
@@ -3234,7 +3242,9 @@ func _guestResourceCountQuery(
 
 	q = q.LeftJoin(isoDevSubQuery, sqlchemy.Equals(isoDevSubQuery.Field("guest_id"), guests.Field("id")))
 
-	q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	if len(rangeObjs) > 0 || len(hostTypes) > 0 || len(resourceTypes) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	}
 
 	switch scope {
 	case rbacutils.ScopeSystem:
@@ -4701,8 +4711,12 @@ func (self *SGuest) isInReconcile(userCred mcclient.TokenCredential) bool {
 	return false
 }
 
-func (self *SGuest) GetEip() (*SElasticip, error) {
-	return ElasticipManager.getEipForInstance(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id)
+func (self *SGuest) GetEipOrPublicIp() (*SElasticip, error) {
+	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id, "")
+}
+
+func (self *SGuest) GetElasticIp() (*SElasticip, error) {
+	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id, api.EIP_MODE_STANDALONE_EIP)
 }
 
 func (self *SGuest) GetPublicIp() (*SElasticip, error) {
@@ -4712,7 +4726,7 @@ func (self *SGuest) GetPublicIp() (*SElasticip, error) {
 func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extEip cloudprovider.ICloudEIP, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult {
 	result := compare.SyncResult{}
 
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		result.Error(fmt.Errorf("getEip error %s", err))
 		return result
@@ -4856,7 +4870,7 @@ func (self *SGuest) DetachScheduledTask(ctx context.Context, userCred mcclient.T
 }
 
 func (self *SGuest) DeleteEip(ctx context.Context, userCred mcclient.TokenCredential) error {
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		log.Errorf("Delete eip fail for get Eip %s", err)
 		return err
@@ -5157,7 +5171,7 @@ func (self *SGuest) toCreateInput() *api.ServerCreateInput {
 	if host := self.GetHost(); host != nil {
 		r.ResourceType = host.ResourceType
 	}
-	if eip, _ := self.GetEip(); eip != nil && eip.Mode == api.EIP_MODE_STANDALONE_EIP {
+	if eip, _ := self.GetEipOrPublicIp(); eip != nil && eip.Mode == api.EIP_MODE_STANDALONE_EIP {
 		r.EipBw = eip.Bandwidth
 		r.EipChargeType = eip.ChargeType
 	}
@@ -5331,7 +5345,7 @@ func (self *SGuest) getGuestUsage(guestCount int) (SQuota, SRegionQuota, error)
 	}
 	regionUsage.Port = netCount
 	// regionUsage.Bw = self.getBandwidth(false)
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil && errors.Cause(err) != sql.ErrNoRows {
 		return usage, regionUsage, err
 	}

pkg/compute/models/initdb.go

@@ -67,6 +67,8 @@ func InitDB() error {
 		DBInstanceNetworkManager,
 		DBInstanceAccountManager,
 		DBInstanceDatabaseManager,
+
+		SnapshotPolicyDiskManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/loadbalancercachedcertificates.go

@@ -95,7 +95,7 @@ func (self *SCachedLoadbalancerCertificate) ValidateDeleteCondition(ctx context.
 	men := []db.IModelManager{
 		LoadbalancerListenerManager,
 	}
-	lbcertId := self.Id
+	lbcertId := self.CertificateId
 	for _, man := range men {
 		t := man.TableSpec().Instance()
 		pdF := t.Field("pending_deleted")

pkg/compute/models/natgateways.go

@@ -363,7 +363,9 @@ func (self *SNatGateway) SyncWithCloudNatGateway(ctx context.Context, userCred m
 		factory, _ := provider.GetProviderFactory()
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extNat.GetBillingType()
-			self.ExpiredAt = extNat.GetExpiredAt()
+			if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extNat.IsAutoRenew()
 		}
 
@@ -407,7 +409,9 @@ func (manager *SNatGatewayManager) newFromCloudNatGateway(ctx context.Context, u
 	factory, _ := provider.GetProviderFactory()
 	if factory.IsSupportPrepaidResources() {
 		nat.BillingType = extNat.GetBillingType()
-		nat.ExpiredAt = extNat.GetExpiredAt()
+		if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+			nat.ExpiredAt = expired
+		}
 		nat.AutoRenew = extNat.IsAutoRenew()
 	}
 

pkg/compute/models/purge.go

@@ -150,12 +150,7 @@ func (disk *SDisk) purge(ctx context.Context, userCred mcclient.TokenCredential)
 	lockman.LockObject(ctx, disk)
 	defer lockman.ReleaseObject(ctx, disk)
 
-	err := disk.DetachAllSnapshotpolicies(ctx, userCred)
-	if err != nil {
-		return errors.Wrap(err, "disk.DetachAllSnapshotpolicies")
-	}
-
-	err = disk.ValidatePurgeCondition(ctx)
+	err := disk.ValidatePurgeCondition(ctx)
 	if err != nil {
 		return err
 	}

pkg/compute/models/secgroupcache.go

@@ -246,18 +246,45 @@ func (manager *SSecurityGroupCacheManager) FetchCustomizeColumns(
 	manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	regRows := manager.SCloudregionResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 
+	cacheIds := make([]string, len(objs))
+	secIds := make([]string, len(objs))
+	vpcIds := make([]string, len(objs))
 	for i := range rows {
 		rows[i] = api.SecurityGroupCacheDetails{
 			StatusStandaloneResourceDetails: stdRows[i],
 			ManagedResourceInfo:             manRows[i],
 			CloudregionResourceInfo:         regRows[i],
 		}
-		vpc, _ := objs[i].(*SSecurityGroupCache).GetVpc()
-		if vpc != nil {
-			rows[i].Vpc = vpc.Name
+		cache := objs[i].(*SSecurityGroupCache)
+		cacheIds[i] = cache.Id
+		vpcIds[i] = cache.VpcId
+		secIds[i] = cache.SecgroupId
+	}
+	vpcMaps, _ := db.FetchIdNameMap2(VpcManager, vpcIds)
+	for i := range rows {
+		rows[i].Vpc = vpcMaps[vpcIds[i]]
+	}
+
+	secgroups := make(map[string]SSecurityGroup)
+	err := db.FetchStandaloneObjectsByIds(SecurityGroupManager, secIds, &secgroups)
+	if err != nil {
+		log.Errorf("FetchStandaloneObjectsByIds fail: %v", err)
+		return rows
+	}
+
+	virObjs := make([]interface{}, len(objs))
+	for i := range rows {
+		if secgroup, ok := secgroups[secIds[i]]; ok {
+			virObjs[i] = &secgroup
+			rows[i].ProjectId = secgroup.ProjectId
 		}
 	}
 
+	projRows := SecurityGroupManager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, virObjs, fields, isList)
+	for i := range rows {
+		rows[i].ProjectizedResourceInfo = projRows[i]
+	}
+
 	return rows
 }
 

pkg/compute/models/secgroups.go

@@ -943,6 +943,7 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, err
 	}
 
+	secgroup.Status = api.SECGROUP_STATUS_READY
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId

pkg/compute/models/snapshotpolicy.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -370,8 +371,8 @@ func (sp *SSnapshotPolicy) GetExtraDetails(
 }
 
 func (sp *SSnapshotPolicy) getMoreDetails(out api.SnapshotPolicyDetails) api.SnapshotPolicyDetails {
-	out.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
-	out.TimePoints = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+	out.RepeatWeekdaysDisplay = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+	out.TimePointsDisplay = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
 	out.BindingDiskCount, _ = SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	return out
 }
@@ -727,7 +728,7 @@ func (self *SSnapshotPolicyManager) TimePointsToIntArray(n uint32) []int {
 	return bitmap.Uint2IntArray(n)
 }
 
-func (sp *SSnapshotPolicy) ComputeNextSyncTime(base, lastSyncTime time.Time) time.Time {
+func computeNextSyncTime(weekDays, timePoints []int, base time.Time) time.Time {
 	if base.IsZero() {
 		base = time.Now()
 	}
@@ -737,14 +738,12 @@ func (sp *SSnapshotPolicy) ComputeNextSyncTime(base, lastSyncTime time.Time) tim
 	if baseWeekday == 0 {
 		baseWeekday = 7
 	}
-	weekDays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
 	weekDays = append(weekDays, weekDays[0]+7)
 	index := sort.SearchInts(weekDays, baseWeekday)
 	addDay := weekDays[index] - baseWeekday
 	nextTime := base.AddDate(0, 0, addDay)
 
 	// find timePoint closest to the base
-	timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
 	var newHour int
 	if addDay > 0 {
 		newHour = timePoints[0]
@@ -753,7 +752,7 @@ func (sp *SSnapshotPolicy) ComputeNextSyncTime(base, lastSyncTime time.Time) tim
 		index := sort.SearchInts(timePoints, baseHour)
 		index = index % len(timePoints)
 		if timePoints[index] == baseHour {
-			index = index + 1
+			index = (index + 1) % len(timePoints)
 			newHour = timePoints[index]
 		} else {
 			newHour = timePoints[index]
@@ -761,19 +760,31 @@ func (sp *SSnapshotPolicy) ComputeNextSyncTime(base, lastSyncTime time.Time) tim
 	}
 	nextTime = time.Date(nextTime.Year(), nextTime.Month(), nextTime.Day(), newHour, 0, 0, 0, base.Location())
 
-	if sp.RetentionDays <= 0 {
-		return nextTime
-	}
-	if lastSyncTime.IsZero() {
-		lastSyncTime = base
-	}
-	snapshotRentionExpired := lastSyncTime.AddDate(0, 0, sp.RetentionDays)
-	if snapshotRentionExpired.Before(nextTime) {
-		return snapshotRentionExpired
+	if !nextTime.After(base) {
+		// If the calculated NextSyncTime and base are equal, add 1 hour to base and recursive processing.
+		return computeNextSyncTime(weekDays, timePoints, base.Add(time.Hour))
 	}
 	return nextTime
 }
 
+func (sp *SSnapshotPolicy) ComputeNextSyncTime(base time.Time) time.Time {
+	weekDays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+	timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+	if sp.RetentionDays <= 0 {
+		return computeNextSyncTime(weekDays, timePoints, base)
+	}
+	// A snapshotpolicy takes effect every Monday with keeping snapshot 3 days. So, the snapshots should be synchronized every Monday (snapshots) and Thursdays (release snapshots).
+	set := sets.NewInt(weekDays...)
+	for _, day := range weekDays {
+		newDay := (day + sp.RetentionDays) % 7
+		if newDay == 0 {
+			newDay = 7
+		}
+		set.Insert(newDay)
+	}
+	return computeNextSyncTime(set.List(), timePoints, base)
+}
+
 func (sp *SSnapshotPolicy) GenerateCreateSpParams() *cloudprovider.SnapshotPolicyInput {
 	intWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
 	intTimePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)

pkg/compute/models/snapshotpolicy_test.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"testing"
+	"time"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
@@ -70,3 +71,122 @@ func TestSSnapshotPolicy_Key(t *testing.T) {
 		}
 	}
 }
+
+func TestSSnapshotPolicy_ComputeNextSyncTime(t *testing.T) {
+	timeStr := "2006-01-02 15:04:05"
+	t.Run("base test", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{2}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-11-03 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2, 6}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-11-01 02:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+	t.Run("same day", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-10-31 02:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 02:00:00",
+				want: "2020-11-01 02:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 01:00:00",
+				want: "2020-10-31 02:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+	t.Run("retentionday", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  2,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-01 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  8,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-01 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{1, 6}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  4,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-02 04:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+}

pkg/compute/models/snapshotpolicydisks.go

@@ -172,10 +172,28 @@ func (m *SSnapshotPolicyDiskManager) FetchBySnapshotPolicyDisk(spId, diskId stri
 	return &ret[0], nil
 }
 
-func (sdm *SSnapshotPolicyDiskManager) InitalizeData() error {
-	q := sdm.Query().IsNullOrEmpty("next_sync_time")
+func (sdm *SSnapshotPolicyDiskManager) InitializeData() error {
+	diskQ := DiskManager.Query("id").SubQuery()
+	sdQ := sdm.Query().NotIn("disk_id", diskQ)
+
 	var sds []SSnapshotPolicyDisk
-	err := db.FetchModelObjects(sdm, q, &sds)
+	err := db.FetchModelObjects(sdm, sdQ, &sds)
+	if err != nil {
+		return errors.Wrap(err, "unable to FetchModelObjects")
+	}
+	for i := range sds {
+		sd := &sds[i]
+		_, err := db.Update(sd, func() error {
+			return sd.MarkDelete()
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	sds = make([]SSnapshotPolicyDisk, 0)
+	q := sdm.Query().IsNullOrEmpty("next_sync_time")
+	err = db.FetchModelObjects(sdm, q, &sds)
 	if err != nil {
 		return err
 	}
@@ -197,7 +215,7 @@ func (sdm *SSnapshotPolicyDiskManager) InitalizeData() error {
 	for i := range sds {
 		sd := &sds[i]
 		_, err := db.Update(sd, func() error {
-			sd.NextSyncTime = spMap[sd.SnapshotpolicyId].ComputeNextSyncTime(now, now)
+			sd.NextSyncTime = spMap[sd.SnapshotpolicyId].ComputeNextSyncTime(now)
 			return nil
 		})
 		if err != nil {
@@ -457,7 +475,7 @@ func (self *SSnapshotPolicyDiskManager) newSnapshotpolicyDisk(ctx context.Contex
 	spd.SnapshotpolicyId = sp.GetId()
 	spd.DiskId = disk.GetId()
 	now := time.Now()
-	spd.NextSyncTime = sp.ComputeNextSyncTime(now, now)
+	spd.NextSyncTime = sp.ComputeNextSyncTime(now)
 	spd.SetModelManager(self, &spd)
 
 	lockman.LockJointObject(ctx, disk, sp)
@@ -494,6 +512,16 @@ func (self *SSnapshotPolicyDiskManager) ValidateCreateData(ctx context.Context,
 	return data, nil
 }
 
+func (sd *SSnapshotPolicyDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	sp, err := SnapshotPolicyManager.FetchSnapshotPolicyById(sd.SnapshotpolicyId)
+	if err != nil {
+		return err
+	}
+	now := time.Now()
+	sd.NextSyncTime = sp.ComputeNextSyncTime(now)
+	return nil
+}
+
 func (sd *SSnapshotPolicyDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.
 	IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 

pkg/compute/models/snapshots.go

@@ -621,6 +621,11 @@ func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context) error {
 	if count > 0 {
 		return httperrors.NewBadRequestError("snapshot referenced by instance snapshot")
 	}
+	if disk, err := self.GetDisk(); err == nil {
+		if disk.Status == api.DISK_RESET {
+			return httperrors.NewBadRequestError("Cannot delete snapshot on disk reset")
+		}
+	}
 	driver := self.GetRegionDriver()
 	if driver != nil {
 		return driver.ValidateSnapshotDelete(ctx, self)

pkg/compute/options/options.go

@@ -188,5 +188,12 @@ func OnOptionsChange(oldO, newO interface{}) bool {
 	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
 		changed = true
 	}
+
+	if oldOpts.PendingDeleteCheckSeconds != newOpts.PendingDeleteCheckSeconds {
+		if !oldOpts.IsSlaveNode {
+			changed = true
+		}
+	}
+
 	return changed
 }

pkg/compute/tasks/disk_delete_task.go

@@ -179,7 +179,6 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteComplete(ctx context.Context, obj d
 	if len(disk.SnapshotId) > 0 && disk.GetMetadata("merge_snapshot", nil) == "true" {
 		models.SnapshotManager.AddRefCount(disk.SnapshotId, -1)
 	}
-	disk.DetachAllSnapshotpolicies(ctx, self.UserCred)
 	disk.RealDelete(ctx, self.UserCred)
 	self.SetStageComplete(ctx, nil)
 }

pkg/compute/tasks/guest_delete_task.go

@@ -118,7 +118,7 @@ func (self *GuestDeleteTask) OnStartEipDissociate(ctx context.Context, guest *mo
 			sourceGuest.StartSyncstatus(ctx, self.UserCred, "")
 		}
 	}
-	eip, _ := guest.GetEip()
+	eip, _ := guest.GetEipOrPublicIp()
 	if eip != nil && eip.Mode != api.EIP_MODE_INSTANCE_PUBLICIP {
 		// detach floating EIP only
 		if jsonutils.QueryBoolean(self.Params, "purge", false) {

pkg/compute/tasks/guest_detach_disk_task.go

@@ -80,14 +80,6 @@ func (self *GuestDetachDiskTask) OnDetachDiskComplete(ctx context.Context, guest
 		return
 	}
 	disk := objDisk.(*models.SDisk)
-	// detach disk and snapshotpolicy if hypervisor is kvm
-	if guest.Hypervisor == api.HYPERVISOR_KVM {
-		err := disk.DetachAllSnapshotpolicies(ctx, self.UserCred)
-		if err != nil {
-			self.OnTaskFail(ctx, guest, nil, jsonutils.NewString(fmt.Sprintf("detach all snapshotpolicies failed: %s", err.Error())))
-			return
-		}
-	}
 	disk.SetStatus(self.UserCred, api.DISK_READY, "on detach disk complete")
 	keepDisk := jsonutils.QueryBoolean(self.Params, "keep_disk", true)
 	host := guest.GetHost()

pkg/compute/usages/handler.go

@@ -676,7 +676,7 @@ func HostAllUsage(pref string, userCred mcclient.IIdentityProvider, scope rbacut
 	if len(pref) > 0 {
 		prefix = fmt.Sprintf("%s.%s", prefix, pref)
 	}
-	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.None)
+	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.False)
 }
 
 func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope, rangeObjs []db.IStandaloneModel,
@@ -685,7 +685,7 @@ func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, scope rb
 	if len(pref) > 0 {
 		prefix = fmt.Sprintf("%s.%s", prefix, pref)
 	}
-	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.None)
+	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.False)
 }
 
 func BaremetalUsage(userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope, rangeObjs []db.IStandaloneModel,

pkg/esxi/handler/handlers.go

@@ -75,7 +75,7 @@ func uploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		httperrors.MissingParameterError(ctx, w, "miss disk")
 		return
 	}
-	hostutils.DelayTask(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
+	hostutils.DelayTaskWithoutReqctx(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
 	hostutils.ResponseOk(ctx, w)
 }
 

pkg/hostman/guestman/guestman.go

@@ -291,6 +291,11 @@ func (m *SGuestManager) LoadServer(sid string) {
 		log.Errorf("On load server error: %s", err)
 		return
 	}
+
+	if jsonutils.QueryBoolean(guest.Desc, "need_sync_stream_disks", false) {
+		go guest.sendStreamDisksComplete(context.Background())
+	}
+
 	m.CandidateServers[sid] = guest
 }
 

pkg/hostman/guestman/guesttasks.go

@@ -176,6 +176,11 @@ func (d *SGuestDiskSyncTask) syncDisksConf() {
 		d.changeCdrom()
 		return
 	}
+	if idxs := d.guest.GetNeedMergeBackingFileDiskIndexs(); len(idxs) > 0 {
+		d.guest.StreamDisks(context.Background(),
+			func() { d.guest.streamDisksComplete(context.Background()) }, idxs,
+		)
+	}
 	d.callback()
 }
 

pkg/hostman/guestman/qemu-kvm.go

@@ -1271,6 +1271,7 @@ func (s *SKVMGuestInstance) streamDisksComplete(ctx context.Context) {
 		if jsonutils.QueryBoolean(disk, "merge_snapshot", false) {
 			d := disks[i].(*jsonutils.JSONDict)
 			d.Set("merge_snapshot", jsonutils.JSONFalse)
+			s.Desc.Set("need_sync_stream_disks", jsonutils.JSONTrue)
 		}
 	}
 	if err := s.SaveDesc(s.Desc); err != nil {
@@ -1279,10 +1280,24 @@ func (s *SKVMGuestInstance) streamDisksComplete(ctx context.Context) {
 	if err := s.delFlatFiles(ctx); err != nil {
 		log.Errorf("del flat files failed %s", err)
 	}
-	_, err := modules.Servers.PerformAction(hostutils.GetComputeSession(ctx),
-		s.Id, "stream-disks-complete", nil)
-	if err != nil {
-		log.Infof("stream disks complete sync error %s", err)
+	go s.sendStreamDisksComplete(ctx)
+}
+
+func (s *SKVMGuestInstance) sendStreamDisksComplete(ctx context.Context) {
+	for {
+		_, err := modules.Servers.PerformAction(hostutils.GetComputeSession(ctx),
+			s.Id, "stream-disks-complete", nil)
+		if err != nil {
+			log.Errorf("stream disks complete sync error %s", err)
+			time.Sleep(30 * time.Second)
+			continue
+		} else {
+			break
+		}
+	}
+	s.Desc.Remove("need_sync_stream_disks")
+	if err := s.SaveDesc(s.Desc); err != nil {
+		log.Errorf("save guest desc failed %s", err)
 	}
 }
 

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -464,10 +464,10 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 	cmd += fmt.Sprintf(" -machine %s,accel=%s", s.getMachine(), accel)
 	cmd += " -k en-us"
 	// #cmd += " -g 800x600"
-	cmd += fmt.Sprintf(" -smp %d,maxcpus=128", cpu)
+	cmd += fmt.Sprintf(" -smp %d,maxcpus=255", cpu)
 	cmd += fmt.Sprintf(" -name %s", name)
 	// #cmd += fmt.Sprintf(" -uuid %s", self.desc["uuid"])
-	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=262144M", mem)
+	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=524288M", mem)
 
 	if s.manager.host.IsHugepagesEnabled() {
 		cmd += fmt.Sprintf(" -mem-prealloc -mem-path %s", fmt.Sprintf("/dev/hugepages/%s", uuid))

pkg/hostman/hostinfo/hostconsts/hostconsts.go

@@ -1,9 +1,11 @@
 package hostconsts
 
 const (
-	TELEGRAF_TAG_KEY_BRAND     = "brand"
-	TELEGRAF_TAG_KEY_RES_TYPE  = "res_type"
-	TELEGRAF_TAG_KEY_HOST_TYPE = "host_type"
+	TELEGRAF_TAG_KEY_BRAND      = "brand"
+	TELEGRAF_TAG_KEY_PLATFORM   = "platform"
+	TELEGRAF_TAG_KEY_HYPERVISOR = "hypervisor"
+	TELEGRAF_TAG_KEY_RES_TYPE   = "res_type"
+	TELEGRAF_TAG_KEY_HOST_TYPE  = "host_type"
 
 	TELEGRAF_TAG_ONECLOUD_BRAND                = "OneCloud"
 	TELEGRAF_TAG_ONECLOUD_RES_TYPE             = "host"

pkg/hostman/hostinfo/hostinfo.go

@@ -1108,8 +1108,10 @@ func (h *SHostInfo) PutHostOffline() {
 }
 
 func (h *SHostInfo) PutHostOnline() error {
-	if len(h.SysError) > 0 {
+	if len(h.SysError) > 0 && !options.HostOptions.StartHostIgnoreSysError {
 		log.Fatalf("Can't put host online, unless resolve these problem %v", h.SysError)
+	} else if len(h.SysError) > 0 && options.HostOptions.StartHostIgnoreSysError {
+		log.Errorf("Host sys error: %v", h.SysError)
 	}
 
 	if len(h.SysWarning) > 0 {

pkg/hostman/options/options.go

@@ -134,7 +134,8 @@ type SHostOptions struct {
 	HostHealthTimeout   int    `help:"host health timeout" default:"30"`
 	HostLeaseTimeout    int    `help:"lease timeout" default:"10"`
 
-	SyncStorageInfoDurationSecond int `help:"sync storage size duration, unit is second" default:"60"`
+	SyncStorageInfoDurationSecond int  `help:"sync storage size duration, unit is second" default:"60"`
+	StartHostIgnoreSysError       bool `help:"start host agent ignore sys error" default:"false"`
 }
 
 var (

pkg/image/models/image_guest.go

@@ -541,7 +541,7 @@ func (self *SGuestImage) PerformPublic(
 		return nil, errors.Wrap(err, "fail to fetch subimages of guest image")
 	}
 	for i := range images {
-		_, err := images[i].PerformPublic(ctx, userCred, query, input)
+		_, err := images[i].performPublic(ctx, userCred, query, input)
 		if err != nil {
 			return nil, errors.Wrapf(err, "fail to public subimage %s", images[i].GetId())
 		}
@@ -560,7 +560,7 @@ func (self *SGuestImage) PerformPrivate(
 		return nil, errors.Wrap(err, "fail to fetch subimages of guest image")
 	}
 	for i := range images {
-		_, err := images[i].PerformPrivate(ctx, userCred, query, input)
+		_, err := images[i].performPrivate(ctx, userCred, query, input)
 		if err != nil {
 			return nil, errors.Wrapf(err, "fail to private subimage %s", images[i].GetId())
 		}

pkg/image/models/images.go

@@ -1476,21 +1476,29 @@ func (img *SImage) PerformUpdateStatus(ctx context.Context, userCred mcclient.To
 }
 
 func (img *SImage) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) {
-	if img.IsStandard.IsTrue() {
-		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for standard image")
-	}
 	if img.IsGuestImage.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for guest image")
 	}
+	return img.performPublic(ctx, userCred, query, input)
+}
+
+func (img *SImage) performPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) {
+	if img.IsStandard.IsTrue() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for standard image")
+	}
 	return img.SSharableVirtualResourceBase.PerformPublic(ctx, userCred, query, input)
 }
 
-func (img *SImage) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
+func (img *SImage) performPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	if img.IsStandard.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform private for standard image")
 	}
+	return img.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input)
+}
+
+func (img *SImage) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	if img.IsGuestImage.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform private for guest image")
 	}
-	return img.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input)
+	return img.performPrivate(ctx, userCred, query, input)
 }

pkg/image/options/options.go

@@ -56,5 +56,11 @@ func OnOptionsChange(oldO, newO interface{}) bool {
 		changed = true
 	}
 
+	if oldOpts.PendingDeleteCheckSeconds != newOpts.PendingDeleteCheckSeconds {
+		if !oldOpts.IsSlaveNode {
+			changed = true
+		}
+	}
+
 	return changed
 }

pkg/keystone/models/configs.go

@@ -401,9 +401,6 @@ func saveConfigs(userCred mcclient.TokenCredential, action string, model db.IMod
 			return errors.Wrap(err, "SensitiveConfigManager.syncConfig")
 		}
 	}
-	if userCred == nil {
-		userCred = getDefaultAdminCred()
-	}
 	maskedValue := jsonutils.NewString("*")
 	for i := range changedSensitive {
 		if changedSensitive[i].OValue != nil {
@@ -416,8 +413,12 @@ func saveConfigs(userCred mcclient.TokenCredential, action string, model db.IMod
 	changed = append(changed, changedSensitive...)
 	if len(changed) > 0 {
 		notes := jsonutils.Marshal(changed)
+		if userCred == nil {
+			userCred = getDefaultAdminCred()
+		} else {
+			logclient.AddSimpleActionLog(model, logclient.ACT_CHANGE_CONFIG, notes, userCred, true)
+		}
 		db.OpsLog.LogEvent(model, db.ACT_CHANGE_CONFIG, notes, userCred)
-		logclient.AddSimpleActionLog(model, logclient.ACT_CHANGE_CONFIG, notes, userCred, true)
 	}
 	return nil
 }
@@ -495,7 +496,7 @@ func uploadConfig(service *SService, config jsonutils.JSONObject) {
 	if service.isCommonService() {
 		err = saveConfigs(nil, "", service, tconf, api.CommonWhitelistOptionMap, nil, nil)
 	} else {
-		err = saveConfigs(nil, "", service, tconf, nil, api.ServiceBlacklistOptionMap, nil)
+		err = saveConfigs(nil, "", service, tconf, nil, api.MergeServiceConfigOptions(api.CommonWhitelistOptionMap, api.ServiceBlacklistOptionMap), nil)
 	}
 	if err != nil {
 		log.Errorf("saveConfigs fail %s", err)

pkg/keystone/models/endpoints.go

@@ -20,6 +20,8 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"sync"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -47,6 +49,7 @@ type SEndpointManager struct {
 	SRegionResourceBaseManager
 
 	informerBackends map[string]informer.IInformerBackend
+	informerSetter   sync.Once
 }
 
 var EndpointManager *SEndpointManager
@@ -131,7 +134,8 @@ func (manager *SEndpointManager) SetInformerBackend() error {
 	if err != nil {
 		return errors.Wrap(err, "fetch informer endpoint")
 	}
-	return manager.SetInformerBackendByEndpoint(informerEp)
+	manager.SetInformerBackendUntilSuccess(informerEp)
+	return nil
 }
 
 func (manager *SEndpointManager) getSessionEndpointType() string {
@@ -160,11 +164,30 @@ func (manager *SEndpointManager) IsEtcdInformerBackend(ep *SEndpoint) bool {
 	return true
 }
 
-func (manager *SEndpointManager) SetInformerBackendByEndpoint(ep *SEndpoint) error {
-	if !manager.IsEtcdInformerBackend(ep) {
-		return nil
+func (manager *SEndpointManager) SetInformerBackendUntilSuccess(ep *SEndpoint) {
+	if informer.GetDefaultBackend() != nil {
+		log.Infof("Informer backend has been setted")
+		return
 	}
-	return manager.SetEtcdInformerBackend(ep)
+	manager.informerSetter.Do(func() {
+		go func() {
+			for {
+				if err := manager.SetEtcdInformerBackend(ep); err != nil {
+					log.Errorf("Set etcd informer backend failed: %s", err)
+					time.Sleep(time.Second * 30)
+				} else {
+					break
+				}
+			}
+		}()
+	})
+}
+
+func (manager *SEndpointManager) SetInformerBackendByEndpoint(ep *SEndpoint) {
+	if !manager.IsEtcdInformerBackend(ep) {
+		return
+	}
+	manager.SetInformerBackendUntilSuccess(ep)
 }
 
 func (manager *SEndpointManager) fetchInformerEndpoint() (*SEndpoint, error) {
@@ -587,9 +610,7 @@ func (manager *SEndpointManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fie
 }
 
 func (endpoint *SEndpoint) trySetInformerBackend() {
-	if err := EndpointManager.SetInformerBackendByEndpoint(endpoint); err != nil {
-		log.Errorf("Set informer by endpoint %s error: %v", endpoint.GetName(), err)
-	}
+	EndpointManager.SetInformerBackendByEndpoint(endpoint)
 }
 
 func (endpoint *SEndpoint) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {

pkg/keystone/models/passwords.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	o "yunion.io/x/onecloud/pkg/keystone/options"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 // +onecloud:swagger-gen-ignore
@@ -111,6 +112,15 @@ func validatePasswordComplexity(password string) error {
 	if o.Options.PasswordMinimalLength > 0 && len(password) < o.Options.PasswordMinimalLength {
 		return errors.Wrap(httperrors.ErrWeakPassword, "too simple password")
 	}
+	if o.Options.PasswordCharComplexity > 0 {
+		complexity := o.Options.PasswordCharComplexity
+		if complexity > 4 {
+			complexity = 4
+		}
+		if stringutils2.GetCharTypeCount(password) < complexity {
+			return errors.Wrap(httperrors.ErrWeakPassword, "too simple password")
+		}
+	}
 	return nil
 }
 

pkg/keystone/models/services.go

@@ -203,7 +203,7 @@ func (service *SService) PerformConfig(ctx context.Context, userCred mcclient.To
 	if service.isCommonService() {
 		err = saveConfigs(userCred, action, service, opts, api.CommonWhitelistOptionMap, nil, nil)
 	} else {
-		err = saveConfigs(userCred, action, service, opts, nil, api.ServiceBlacklistOptionMap, nil)
+		err = saveConfigs(userCred, action, service, opts, nil, api.MergeServiceConfigOptions(api.CommonWhitelistOptionMap, api.ServiceBlacklistOptionMap), nil)
 	}
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("saveConfig fail %s", err)

pkg/keystone/options/options.go

@@ -41,6 +41,7 @@ type SKeystoneOptions struct {
 	PasswordExpirationSeconds  int `help:"password expires after the duration in seconds"`
 	PasswordMinimalLength      int `help:"password minimal length" default:"6"`
 	PasswordUniqueHistoryCheck int `help:"password must be unique in last N passwords"`
+	PasswordCharComplexity     int `help:"password complexity policy" default:"0"`
 
 	PasswordErrorLockCount int `help:"lock user account if given number of failed auth"`
 

pkg/mcclient/options/servers.go

@@ -71,6 +71,8 @@ type ServerListOptions struct {
 	WithMeta *bool `help:"filter by metadata" negative:"without_meta"`
 
 	WithUserMeta *bool `help:"filter by user metadata" negative:"without_user_meta"`
+
+	WithHost *bool `help:"filter guest with host or not" negative:"without_host"`
 }
 
 func (o *ServerListOptions) Params() (jsonutils.JSONObject, error) {

pkg/monitor/alerting/notifier.go

@@ -221,5 +221,9 @@ func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule {
 		alertRule.MeasurementDesc = evalCtx.EvalMatches[0].MeasurementDesc
 		alertRule.FieldDesc = evalCtx.EvalMatches[0].FieldDesc
 	}
+	if len(evalCtx.AlertOkEvalMatches) != 0 {
+		alertRule.MeasurementDesc = evalCtx.AlertOkEvalMatches[0].MeasurementDesc
+		alertRule.FieldDesc = evalCtx.AlertOkEvalMatches[0].FieldDesc
+	}
 	return alertRule
 }

pkg/monitor/alertresourcedrivers/cloudaccount.go

@@ -36,10 +36,10 @@ func (drvF cloudaccountDriverF) GetType() monitor.AlertResourceType {
 
 func (drvF cloudaccountDriverF) IsEvalMatched(input monitor.EvalMatch) bool {
 	tags := input.Tags
-	_, hasId := tags[CLOUDACCOUNT_TAG_ID_KEY]
-	if !hasId {
-		return false
-	}
+	//_, hasId := tags[CLOUDACCOUNT_TAG_ID_KEY]
+	//if !hasId {
+	//	return false
+	//}
 	_, hasName := tags[CLOUDACCOUNT_TAG_NAME_KEY]
 	if !hasName {
 		return false

pkg/monitor/models/alertrecord.go

@@ -171,6 +171,11 @@ func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.To
 		log.Errorf("Reconcile from alert record error: %v", err)
 		return
 	}
+	err := GetAlertResourceManager().NotifyAlertResourceCount(ctx)
+	if err != nil {
+		log.Errorf("NotifyAlertResourceCount error: %v", err)
+		return
+	}
 }
 
 func (record *SAlertRecord) GetState() monitor.AlertStateType {

pkg/monitor/models/alertresource.go

@@ -17,6 +17,7 @@ package models
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -27,11 +28,15 @@ import (
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	mc_modules "yunion.io/x/onecloud/pkg/mcclient/modules"
+	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 var (
 	alertResourceManager *SAlertResourceManager
+	adminUsers           *sync.Map
 )
 
 func init() {
@@ -373,3 +378,110 @@ func (res *SAlertResource) CustomizeDelete(
 	}
 	return nil
 }
+
+func (manager *SAlertResourceManager) NotifyAlertResourceCount(ctx context.Context) error {
+	log.Errorln("exec NotifyAlertResourceCount func")
+	cn, err := manager.getResourceCount()
+	if err != nil {
+		return err
+	}
+	alertResourceCount := resourceCount{
+		AlertResourceCount: cn,
+	}
+	if adminUsers == nil {
+		manager.GetAdminRoleUsers(ctx, nil, true)
+	}
+	adminUsersTmp := *adminUsers
+	ids := make([]string, 0)
+	adminUsersTmp.Range(func(key, value interface{}) bool {
+		ids = append(ids, key.(string))
+		return true
+	})
+	if len(ids) == 0 {
+		return fmt.Errorf("no find users in receivers has admin role")
+	}
+	//if len(ids) != 0 {
+	//	notifyclient.RawNotifyWithCtx(ctx, ids, false, npk.NotifyByWebConsole, npk.NotifyPriorityCritical,
+	//		"alertResourceCount", jsonutils.Marshal(&alertResourceCount))
+	//	return nil
+	//} else {
+	//	return fmt.Errorf("no find users in receivers has admin role")
+	//}
+	manager.sendWebsocketInfo(ids, alertResourceCount)
+	return nil
+}
+
+type resourceCount struct {
+	AlertResourceCount int `json:"alert_resource_count"`
+}
+
+func (manager *SAlertResourceManager) getResourceCount() (int, error) {
+	query := manager.Query("id")
+	cn, err := query.CountWithError()
+	if err != nil {
+		return cn, errors.Wrap(err, "SAlertResourceManager get resource count error")
+	}
+
+	return cn, nil
+}
+
+func (manager *SAlertResourceManager) GetAdminRoleUsers(ctx context.Context, userCred mcclient.TokenCredential,
+	isStart bool) {
+	if adminUsers == nil {
+		adminUsers = new(sync.Map)
+	}
+	offset := 0
+	query := jsonutils.NewDict()
+	session := auth.GetAdminSession(ctx, "", "")
+	rid, err := mc_modules.RolesV3.GetId(session, "admin", jsonutils.NewDict())
+	if err != nil {
+		errors.Errorf("get role id error:%v", err)
+		return
+	}
+	query.Add(jsonutils.NewString(rid), "role", "id")
+	for {
+		query.Set("offset", jsonutils.NewInt(int64(offset)))
+		result, err := mc_modules.RoleAssignments.List(session, query)
+		if err != nil {
+			errors.Errorf("get admin role list error:%v", err)
+			return
+		}
+		for _, roleAssign := range result.Data {
+			userId, err := roleAssign.GetString("user", "id")
+			if err != nil {
+				log.Errorf("roleAssign:%v", roleAssign)
+				return
+			}
+			//_, err = mc_modules.NotifyReceiver.GetById(session, userId, jsonutils.NewDict())
+			//if err != nil {
+			//	log.Errorf("Recipients GetById err:%v", err)
+			//	continue
+			//}
+			adminUsers.Store(userId, roleAssign)
+		}
+		offset = result.Offset + len(result.Data)
+		if offset >= result.Total {
+			break
+		}
+	}
+}
+
+func (manager *SAlertResourceManager) sendWebsocketInfo(uids []string, alertResourceCount resourceCount) {
+	session := auth.GetAdminSession(context.Background(), "", "")
+	params := jsonutils.NewDict()
+	params.Set("obj_type", jsonutils.NewString("monitor"))
+	params.Set("obj_id", jsonutils.NewString(""))
+	params.Set("obj_name", jsonutils.NewString(""))
+	params.Set("success", jsonutils.JSONTrue)
+	params.Set("action", jsonutils.NewString("alertResourceCount"))
+	params.Set("notes", jsonutils.NewString(fmt.Sprintf("priority=%s; content=%s", string(npk.NotifyPriorityCritical),
+		jsonutils.Marshal(&alertResourceCount).String())))
+	for _, uid := range uids {
+		params.Set("user_id", jsonutils.NewString(uid))
+		params.Set("user", jsonutils.NewString(uid))
+		_, err := mc_modules.Websockets.Create(session, params)
+		if err != nil {
+			log.Errorf("websocket send info err:%v", err)
+		}
+	}
+}

pkg/monitor/models/commonalert.go

@@ -706,9 +706,11 @@ func (alert *SCommonAlert) ValidateUpdateData(
 		if err != nil {
 			return data, errors.Wrap(err, "metric query error")
 		}
-
 		if alert.getAlertType() == monitor.CommonAlertSystemAlertType {
-			return data, nil
+			forceUpdate, _ := data.Bool("force_update")
+			if !forceUpdate {
+				return data, nil
+			}
 		}
 		data.Update(jsonutils.Marshal(metricQuery))
 		err = data.Unmarshal(updataInput)

pkg/monitor/models/datasource.go

@@ -34,9 +34,9 @@ import (
 	"yunion.io/x/pkg/util/wait"
 	"yunion.io/x/pkg/utils"
 
-	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	merrors "yunion.io/x/onecloud/pkg/monitor/errors"
@@ -94,25 +94,32 @@ func (man *SDataSourceManager) Run(ctx context.Context) error {
 
 func (man *SDataSourceManager) initDefaultDataSource(ctx context.Context) error {
 	region := options.Options.Region
+	epType := options.Options.SessionEndpointType
 	initF := func() {
 		ds, err := man.GetDefaultSource()
 		if err != nil && err != ErrDataSourceDefaultNotFound {
 			log.Errorf("Get default datasource: %v", err)
 			return
 		}
-		if ds != nil {
-			return
-		}
-		s := auth.GetAdminSessionWithPublic(ctx, region, "")
+		s := auth.GetAdminSession(ctx, region, "")
 		if s == nil {
 			log.Errorf("get empty public session for region %s", region)
 			return
 		}
-		url, err := s.GetServiceURL("influxdb", identityapi.EndpointInterfacePublic)
+		url, err := s.GetServiceURL("influxdb", epType)
 		if err != nil {
 			log.Errorf("get influxdb public url: %v", err)
 			return
 		}
+		if ds != nil {
+			if _, err := db.Update(ds, func() error {
+				ds.Url = url
+				return nil
+			}); err != nil {
+				log.Errorf("update datasource url error: %v", err)
+			}
+			return
+		}
 		ds = &SDataSource{
 			Type: monitor.DataSourceTypeInfluxdb,
 			Url:  url,
@@ -608,10 +615,34 @@ func (self *SDataSourceManager) GetMetricMeasurement(query jsonutils.JSONObject,
 	if err != nil {
 		return jsonutils.JSONNull, errors.Wrap(err, "getTagValue error")
 	}
+	self.filterRtnTags(output)
 	return jsonutils.Marshal(output), nil
 
 }
 
+func (self *SDataSourceManager) filterRtnTags(output *monitor.InfluxMeasurement) {
+	for _, tag := range []string{hostconsts.TELEGRAF_TAG_KEY_BRAND, hostconsts.TELEGRAF_TAG_KEY_PLATFORM,
+		hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR} {
+		if val, ok := output.TagValue[tag]; ok {
+			output.TagValue[hostconsts.TELEGRAF_TAG_KEY_BRAND] = val
+			break
+		}
+	}
+	for _, tag := range []string{"source", "status", hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE,
+		hostconsts.TELEGRAF_TAG_KEY_RES_TYPE, "is_vm", "os_type", hostconsts.TELEGRAF_TAG_KEY_PLATFORM,
+		hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR, "domain_name", "region"} {
+		if _, ok := output.TagValue[tag]; ok {
+			delete(output.TagValue, tag)
+		}
+	}
+
+	repTag := make([]string, 0)
+	for tag, _ := range output.TagValue {
+		repTag = append(repTag, tag)
+	}
+	output.TagKey = repTag
+}
+
 func (self *SDataSourceManager) filterTagValue(measurement monitor.InfluxMeasurement, timeF timeFilter,
 	db *influxdb.SInfluxdb, tagValChan *influxdbTagValueChan, tagFilter string) error {
 	ctx, _ := context.WithTimeout(context.Background(), time.Second*5)

pkg/monitor/models/unifiedmonitor.go

@@ -11,6 +11,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	merrors "yunion.io/x/onecloud/pkg/monitor/errors"
@@ -392,6 +393,12 @@ func fillSerieTags(series *tsdb.TimeSeriesSlice) {
 				break
 			}
 		}
+		for _, tag := range []string{"source", "status", hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE,
+			hostconsts.TELEGRAF_TAG_KEY_RES_TYPE, "cpu", "is_vm", "os_type", "domain_name", "region"} {
+			if _, ok := serie.Tags[tag]; ok {
+				delete(serie.Tags, tag)
+			}
+		}
 		(*series)[i] = serie
 	}
 }

pkg/monitor/options/options.go

@@ -22,12 +22,13 @@ type AlerterOptions struct {
 	common_options.CommonOptions
 	common_options.DBOptions
 
-	DataProxyTimeout                      int   `help:"query data source proxy timeout" default:"30"`
-	AlertingMinIntervalSeconds            int64 `help:"alerting min schedule frequency" default:"10"`
-	AlertingMaxAttempts                   int   `help:"alerting engine max attempt" default:"3"`
-	AlertingEvaluationTimeoutSeconds      int64 `help:"alerting evaluation timeout" default:"5"`
-	AlertingNotificationTimeoutSeconds    int64 `help:"alerting notification timeout" default:"30"`
-	InitScopeSuggestConfigIntervalSeconds int   `help:"internal to init scope suggest configs" default:"900"`
+	DataProxyTimeout                               int   `help:"query data source proxy timeout" default:"30"`
+	AlertingMinIntervalSeconds                     int64 `help:"alerting min schedule frequency" default:"10"`
+	AlertingMaxAttempts                            int   `help:"alerting engine max attempt" default:"3"`
+	AlertingEvaluationTimeoutSeconds               int64 `help:"alerting evaluation timeout" default:"5"`
+	AlertingNotificationTimeoutSeconds             int64 `help:"alerting notification timeout" default:"30"`
+	InitScopeSuggestConfigIntervalSeconds          int   `help:"internal to init scope suggest configs" default:"900"`
+	InitAlertResourceAdminRoleUsersIntervalSeconds int   `help:"internal to init alert resource admin role users " default:"3600"`
 }
 
 var (

pkg/monitor/service/handlers.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"strconv"
 
 	"github.com/gorilla/mux"
@@ -120,4 +121,13 @@ func addMiscHandlers(root *mux.Router) {
 		}
 	}
 	root.HandleFunc("/subscriptions/write", adapterF(performHandler))
+
+	// ref: pkg/appsrv/appsrv:addDefaultHandlers
+	root.HandleFunc("/version", adapterF(appsrv.VersionHandler))
+	root.HandleFunc("/stats", adapterF(appsrv.StatisticHandler))
+	root.HandleFunc("/ping", adapterF(appsrv.PingHandler))
+	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
+
+	// pprof handler
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }

pkg/monitor/service/service.go

@@ -66,6 +66,7 @@ func StartService() {
 	cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount)
 	suggestsysdrivers.InitSuggestSysRuleCronjob()
 	cron.AddJobAtIntervalsWithStartRun("InitScopeSuggestConfigs", time.Duration(opts.InitScopeSuggestConfigIntervalSeconds)*time.Second, models.SuggestSysRuleConfigManager.InitScopeConfigs, true)
+	cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true)
 	cron.Start()
 	defer cron.Stop()
 

pkg/multicloud/aliyun/dns_domain.go

@@ -249,6 +249,9 @@ func (self *SDomain) GetId() string {
 }
 
 func (self *SDomain) GetName() string {
+	if len(self.PunyCode) > 0 {
+		return self.PunyCode
+	}
 	return self.DomainName
 }
 

pkg/multicloud/aliyun/latitud_and_longitude.go

@@ -24,6 +24,7 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
 	"cn-beijing":            {Latitude: 39.904202, Longitude: 116.407394, City: api.CITY_BEI_JING, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-zhangjiakou":        {Latitude: 40.767544, Longitude: 114.886337, City: api.CITY_ZHANG_JIA_KOU, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-huhehaote":          {Latitude: 40.842358, Longitude: 111.749992, City: api.CITY_HU_HE_HAO_TE, CountryCode: api.COUNTRY_CODE_CN},
+	"cn-huhehaote-nebula-1": {Latitude: 40.842358, Longitude: 111.749992, City: api.CITY_HU_HE_HAO_TE, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-hangzhou":           {Latitude: 30.274084, Longitude: 120.155067, City: api.CITY_HANG_ZHOU, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-shanghai":           {Latitude: 31.230391, Longitude: 121.473701, City: api.CITY_SHANG_HAI, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-shanghai-finance-1": {Latitude: 31.230391, Longitude: 121.473701, City: api.CITY_SHANG_HAI, CountryCode: api.COUNTRY_CODE_CN},
@@ -44,4 +45,5 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
 	"me-east-1":             {Latitude: 25.204849, Longitude: 55.270782, City: api.CITY_DUBAI, CountryCode: api.COUNTRY_CODE_AE},
 	"eu-central-1":          {Latitude: 50.110924, Longitude: 8.682127, City: api.CITY_FRANKFURT, CountryCode: api.COUNTRY_CODE_DE},
 	"cn-wulanchabu":         {Latitude: 41.0178065, Longitude: 113.094978, City: api.CITY_WU_LAN_CHA_BU, CountryCode: api.COUNTRY_CODE_CN},
+	"cn-guangzhou":          {Latitude: 23.129110, Longitude: 113.264381, City: api.CITY_GUANG_ZHOU, CountryCode: api.COUNTRY_CODE_CN},
 }

pkg/multicloud/aws/aws.go

@@ -377,7 +377,7 @@ func (self *SAwsClient) GetIHostById(id string) (cloudprovider.ICloudHost, error
 		ihost, err := self.iregions[i].GetIHostById(id)
 		if err == nil {
 			return ihost, nil
-		} else if err != cloudprovider.ErrNotFound {
+		} else if errors.Cause(err) != cloudprovider.ErrNotFound {
 			return nil, err
 		}
 	}
@@ -389,7 +389,7 @@ func (self *SAwsClient) GetIVpcById(id string) (cloudprovider.ICloudVpc, error)
 		ihost, err := self.iregions[i].GetIVpcById(id)
 		if err == nil {
 			return ihost, nil
-		} else if err != cloudprovider.ErrNotFound {
+		} else if errors.Cause(err) != cloudprovider.ErrNotFound {
 			return nil, err
 		}
 	}
@@ -401,7 +401,7 @@ func (self *SAwsClient) GetIStorageById(id string) (cloudprovider.ICloudStorage,
 		ihost, err := self.iregions[i].GetIStorageById(id)
 		if err == nil {
 			return ihost, nil
-		} else if err != cloudprovider.ErrNotFound {
+		} else if errors.Cause(err) != cloudprovider.ErrNotFound {
 			return nil, err
 		}
 	}

pkg/multicloud/aws/region.go

@@ -523,7 +523,7 @@ func (self *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) {
 		ihost, err := izones[i].GetIHostById(id)
 		if err == nil {
 			return ihost, nil
-		} else if err != cloudprovider.ErrNotFound {
+		} else if errors.Cause(err) != cloudprovider.ErrNotFound {
 			return nil, err
 		}
 	}
@@ -539,7 +539,7 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er
 		istore, err := izones[i].GetIStorageById(id)
 		if err == nil {
 			return istore, nil
-		} else if err != cloudprovider.ErrNotFound {
+		} else if errors.Cause(err) != cloudprovider.ErrNotFound {
 			return nil, err
 		}
 	}

pkg/multicloud/esxi/host.go

@@ -966,6 +966,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 		}
 	}
 
+	var rootDiskSizeMb int64
 	if len(params.Disks) > 0 {
 		driver := params.Disks[0].Driver
 		if driver == "scsi" || driver == "pvscsi" {
@@ -993,30 +994,10 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 			}
 		}
 
-		// resize system disk
-		sysDiskSize := params.Disks[0].Size
-		if sysDiskSize == 0 {
-			sysDiskSize = 30 * 1024
+		rootDiskSizeMb = params.Disks[0].Size
+		if rootDiskSizeMb == 0 {
+			rootDiskSizeMb = 30 * 1024
 		}
-		if int64(from.vdisks[0].GetDiskSizeMB()) != sysDiskSize {
-			vdisk := from.vdisks[0].getVirtualDisk()
-			vdisk.CapacityInKB = sysDiskSize * 1024
-			spec := &types.VirtualDeviceConfigSpec{}
-			spec.Operation = types.VirtualDeviceConfigSpecOperationEdit
-			spec.Device = vdisk
-			deviceChange = append(deviceChange, spec)
-			log.Infof("resize system disk: %dGB => %dGB", from.vdisks[0].GetDiskSizeMB()/1024, vdisk.CapacityInKB/1024/1024)
-		}
-		// remove extra disk
-		// for i := 1; i < len(from.vdisks); i++ {
-		// 	 dev := from.vdisks[i].dev
-		// 	 spec := &types.VirtualDeviceConfigSpec{}
-		// 	 spec.Operation = types.VirtualDeviceConfigSpecOperationRemove
-		// 	 spec.Device = dev
-		//	 spec.FileOperation = types.VirtualDeviceConfigSpecFileOperationDestroy
-		//	 deviceChange = append(deviceChange, spec)
-		//	 log.Debugf("remove disk, index: %d", i)
-		// }
 	}
 
 	dc, err := host.GetDatacenter()
@@ -1081,6 +1062,13 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 	if vm == nil {
 		return nil, errors.Error("clone successfully but unable to NewVirtualMachine")
 	}
+	// resize system disk
+	if rootDiskSizeMb > 0 && int64(vm.vdisks[0].GetDiskSizeMB()) != rootDiskSizeMb {
+		err = vm.vdisks[0].Resize(ctx, rootDiskSizeMb)
+		if err != nil {
+			return vm, errors.Wrap(err, "resize for root disk")
+		}
+	}
 	// add data disk
 	for i := 1; i < len(params.Disks); i++ {
 		size := params.Disks[i].Size

pkg/multicloud/esxi/virtualmachine.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/vmware/govmomi/nfc"
 	"github.com/vmware/govmomi/object"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
@@ -730,8 +731,13 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		return fmt.Errorf("invalid vm")
 	}
 
-	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
-		dev := moVM.Config.Hardware.Device[i]
+	// sort devices via their Key
+	devices := moVM.Config.Hardware.Device
+	sort.Slice(devices, func(i, j int) bool {
+		return devices[i].GetVirtualDevice().Key < devices[j].GetVirtualDevice().Key
+	})
+	for i := 0; i < len(devices); i += 1 {
+		dev := devices[i]
 		devType := reflect.Indirect(reflect.ValueOf(dev)).Type()
 
 		etherType := reflect.TypeOf((*types.VirtualEthernetCard)(nil)).Elem()
@@ -742,7 +748,7 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		if reflectutils.StructContains(devType, etherType) {
 			self.vnics = append(self.vnics, NewVirtualNIC(self, dev, len(self.vnics)))
 		} else if reflectutils.StructContains(devType, diskType) {
-			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vnics)))
+			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vdisks)))
 		} else if reflectutils.StructContains(devType, vgaType) {
 			self.vga = NewVirtualVGA(self, dev, 0)
 		} else if reflectutils.StructContains(devType, cdromType) {
@@ -1175,8 +1181,19 @@ func (self *SVirtualMachine) ExportTemplate(ctx context.Context, idx int, diskPa
 	lr := newLeaseLogger("download vmdk", 5)
 	lr.Log()
 	defer lr.End()
+
+	// filter vmdk item
+	vmdkItems := make([]nfc.FileItem, 0, len(info.Items)/2)
+	for i := range info.Items {
+		if strings.HasSuffix(info.Items[i].Path, ".vmdk") {
+			vmdkItems = append(vmdkItems, info.Items[i])
+		} else {
+			log.Infof("item.Path does not end in '.vmdk': %#v", info.Items[i])
+		}
+	}
+
 	log.Debugf("download to %s start...", diskPath)
-	err = lease.DownloadFile(ctx, diskPath, info.Items[idx], soap.Download{Progress: lr})
+	err = lease.DownloadFile(ctx, diskPath, vmdkItems[idx], soap.Download{Progress: lr})
 	if err != nil {
 		return errors.Wrap(err, "lease.DownloadFile")
 	}

pkg/multicloud/qcloud/dnspod_domain.go

@@ -58,10 +58,14 @@ type SDomian struct {
 }
 
 // https://cloud.tencent.com/document/product/302/8505
-func (client *SQcloudClient) GetDomains(offset int, limit int) ([]SDomian, int, error) {
+func (client *SQcloudClient) GetDomains(projectId string, offset int, limit int) ([]SDomian, int, error) {
 	params := map[string]string{}
 	params["offset"] = strconv.Itoa(offset)
 	params["length"] = strconv.Itoa(limit)
+	if len(projectId) > 0 {
+		params["qProjectId"] = projectId
+	}
+
 	resp, err := client.cnsRequest("DomainList", params)
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "client.cnsRequest(DomainList, %s)", jsonutils.Marshal(params).String())
@@ -87,7 +91,8 @@ func (client *SQcloudClient) GetAllDomains() ([]SDomian, error) {
 	count := 0
 	result := []SDomian{}
 	for {
-		domains, total, err := client.GetDomains(count, 100)
+		// -1 所有项目; 0,default默认项目
+		domains, total, err := client.GetDomains("-1", count, 100)
 		if err != nil {
 			return nil, errors.Wrap(err, " client.GetDomains(count, 100)")
 		}
@@ -168,6 +173,9 @@ func (self *SDomian) GetId() string {
 }
 
 func (self *SDomian) GetName() string {
+	if len(self.Punycode) > 0 {
+		return self.Punycode
+	}
 	return self.Name
 }
 

pkg/multicloud/qcloud/dnspod_record.go

@@ -57,12 +57,15 @@ type SDnsRecord struct {
 }
 
 // https://cloud.tencent.com/document/product/302/8517
-func (client *SQcloudClient) GetDnsRecords(sDomainName string, offset int, limit int) ([]SDnsRecord, int, error) {
+func (client *SQcloudClient) GetDnsRecords(projectId string, sDomainName string, offset int, limit int) ([]SDnsRecord, int, error) {
 
 	params := map[string]string{}
 	params["offset"] = strconv.Itoa(offset)
 	params["length"] = strconv.Itoa(limit)
 	params["domain"] = sDomainName
+	if len(projectId) > 0 {
+		params["qProjectId"] = projectId
+	}
 	resp, err := client.cnsRequest("RecordList", params)
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "client.cnsRequest(RecordList, %s)", jsonutils.Marshal(params).String())
@@ -88,7 +91,8 @@ func (client *SQcloudClient) GetAllDnsRecords(sDomainName string) ([]SDnsRecord,
 	count := 0
 	result := []SDnsRecord{}
 	for true {
-		records, total, err := client.GetDnsRecords(sDomainName, count, 100)
+		// -1 所有项目; 0,default默认项目
+		records, total, err := client.GetDnsRecords("-1", sDomainName, count, 100)
 		if err != nil {
 			return nil, errors.Wrapf(err, "client.GetDnsRecords(%s,%d,%d)", sDomainName, count, 100)
 		}

pkg/multicloud/qcloud/project.go

@@ -79,6 +79,9 @@ func (client *SQcloudClient) GetProjects() ([]SProject, error) {
 	projects := []SProject{}
 	params := map[string]string{"allList": "1"}
 	resp, err := client.accountRequestRequest("DescribeProject", params)
+	if err != nil {
+		return nil, errors.Wrapf(err, "DescribeProject")
+	}
 	err = resp.Unmarshal(&projects)
 	if err != nil {
 		return nil, errors.Wrap(err, "resp.Unmarshal")

pkg/multicloud/qcloud/region.go

@@ -86,14 +86,13 @@ func (self *SRegion) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadb
 		return nil, err
 	}
 
-	icerts := []cloudprovider.ICloudLoadbalancerCertificate{}
+	certIds := []string{}
 	for _, lb := range lbs {
 		listeners, err := lb.GetLoadbalancerListeners("HTTPS")
 		if err != nil {
 			return nil, err
 		}
 
-		certIds := []string{}
 		for _, listener := range listeners {
 			if len(listener.Certificate.CERTID) > 0 && !utils.IsInStringArray(listener.Certificate.CERTID, certIds) {
 				certIds = append(certIds, listener.Certificate.CERTID)
@@ -113,15 +112,16 @@ func (self *SRegion) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadb
 				}
 			}
 		}
+	}
 
-		for _, cid := range certIds {
-			icert, err := self.GetILoadBalancerCertificateById(cid)
-			if err != nil {
-				return nil, err
-			}
-
-			icerts = append(icerts, icert)
+	icerts := []cloudprovider.ICloudLoadbalancerCertificate{}
+	for _, cid := range certIds {
+		icert, err := self.GetILoadBalancerCertificateById(cid)
+		if err != nil {
+			return nil, err
 		}
+
+		icerts = append(icerts, icert)
 	}
 
 	return icerts, nil

pkg/multicloud/qcloud/shell/dnspod.go

@@ -24,11 +24,12 @@ import (
 
 func init() {
 	type DomianListOptions struct {
-		Offset int
-		Limit  int
+		Offset    int
+		Limit     int
+		ProjectId string
 	}
 	shellutils.R(&DomianListOptions{}, "domain-list", "List domains", func(cli *qcloud.SRegion, args *DomianListOptions) error {
-		domains, total, e := cli.GetClient().GetDomains(args.Offset, args.Limit)
+		domains, total, e := cli.GetClient().GetDomains(args.ProjectId, args.Offset, args.Limit)
 		if e != nil {
 			return e
 		}
@@ -61,12 +62,13 @@ func init() {
 	})
 
 	type DnsRecordListOptions struct {
-		DOMAIN string
-		Offset int
-		Limit  int
+		DOMAIN    string
+		ProjectId string
+		Offset    int
+		Limit     int
 	}
 	shellutils.R(&DnsRecordListOptions{}, "dnsrecord-list", "List dndrecord", func(cli *qcloud.SRegion, args *DnsRecordListOptions) error {
-		records, total, e := cli.GetClient().GetDnsRecords(args.DOMAIN, args.Offset, args.Limit)
+		records, total, e := cli.GetClient().GetDnsRecords(args.ProjectId, args.DOMAIN, args.Offset, args.Limit)
 		if e != nil {
 			return e
 		}
@@ -84,7 +86,7 @@ func init() {
 	}
 	shellutils.R(&DnsRecordCreateOptions{}, "dnsrecord-create", "create dndrecord", func(cli *qcloud.SRegion, args *DnsRecordCreateOptions) error {
 		change := cloudprovider.DnsRecordSet{}
-		change.DnsName = args.DOMAIN
+		change.DnsName = args.NAME
 		change.DnsValue = args.VALUE
 		change.Ttl = args.TTL
 		change.DnsType = cloudprovider.TDnsType(args.TYPE)

pkg/notify/models/receiver.go

@@ -63,6 +63,10 @@ var (
 		api.DINGTALK_ROBOT,
 		api.WORKWX_ROBOT,
 	}
+	AllOkContactTypes = append(AllRobotContactTypes,
+		api.WEBCONSOLE,
+		api.WEBHOOK,
+	)
 )
 
 type SReceiverManager struct {
@@ -264,18 +268,18 @@ func (rm *SReceiverManager) ValidateCreateData(ctx context.Context, userCred mcc
 	// hack
 	input.Name = input.UName
 	// validate email
-	if ok := regutils.MatchEmail(input.Email); !ok {
+	if ok := regutils.MatchEmail(input.Email); len(input.Email) > 0 && !ok {
 		return input, httperrors.NewInputParameterError("invalid email")
 	}
 	// validate mobile
-	if ok := regutils.MatchMobile(input.Mobile); !ok {
+	if ok := regutils.MatchMobile(input.Mobile); len(input.Mobile) > 0 && !ok {
 		return input, httperrors.NewInputParameterError("invalid mobile")
 	}
 	return input, nil
 }
 
 func (r *SReceiver) IsEnabledContactType(ct string) (bool, error) {
-	if utils.IsInStringArray(ct, AllRobotContactTypes) {
+	if utils.IsInStringArray(ct, AllOkContactTypes) {
 		return true, nil
 	}
 	if ct == api.WEBCONSOLE {
@@ -289,7 +293,7 @@ func (r *SReceiver) IsEnabledContactType(ct string) (bool, error) {
 }
 
 func (r *SReceiver) IsVerifiedContactType(ct string) (bool, error) {
-	if utils.IsInStringArray(ct, AllRobotContactTypes) {
+	if utils.IsInStringArray(ct, AllOkContactTypes) {
 		return true, nil
 	}
 	cts, err := r.GetVerifiedContactTypes()

pkg/util/stringutils2/stringutils.go

@@ -185,3 +185,29 @@ func GenerateHostName(name string, osType string) string {
 	}
 	return forOther(name)
 }
+
+func GetCharTypeCount(str string) int {
+	digitIdx := 0
+	lowerIdx := 1
+	upperIdx := 2
+	otherIdx := 3
+	complexity := make([]int, 4)
+	for _, b := range []byte(str) {
+		if b >= '0' && b <= '9' {
+			complexity[digitIdx] += 1
+		} else if b >= 'a' && b <= 'z' {
+			complexity[lowerIdx] += 1
+		} else if b >= 'A' && b <= 'Z' {
+			complexity[upperIdx] += 1
+		} else {
+			complexity[otherIdx] += 1
+		}
+	}
+	ret := 0
+	for i := range complexity {
+		if complexity[i] > 0 {
+			ret += 1
+		}
+	}
+	return ret
+}

pkg/util/stringutils2/stringutils_test.go

@@ -166,3 +166,45 @@ func TestGenerateHostName(t *testing.T) {
 		}
 	}
 }
+
+func TestGetCharTypeCount(t *testing.T) {
+	cases := []struct {
+		in   string
+		want int
+	}{
+		{
+			in:   "",
+			want: 0,
+		},
+		{
+			in:   "123",
+			want: 1,
+		},
+		{
+			in:   "abc",
+			want: 1,
+		},
+		{
+			in:   "abcAbc",
+			want: 2,
+		},
+		{
+			in:   "123dbA",
+			want: 3,
+		},
+		{
+			in:   "123@Acv",
+			want: 4,
+		},
+		{
+			in:   "中文",
+			want: 1,
+		},
+	}
+	for _, c := range cases {
+		got := GetCharTypeCount(c.in)
+		if got != c.want {
+			t.Errorf("GetCharTypeCount %s want %d got %d", c.in, c.want, got)
+		}
+	}
+}

pkg/webconsole/service/service.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
-	"net/http/pprof"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"strconv"
@@ -133,9 +133,5 @@ func addMiscHandlers(root *mux.Router) {
 	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
 
 	// pprof handler
-	root.HandleFunc("/debug/pprof/", pprof.Index)
-	root.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-	root.HandleFunc("/debug/pprof/profile", pprof.Profile)
-	root.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-	root.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }

vendor/modules.txt

@@ -1076,7 +1076,7 @@ yunion.io/x/log/hooks
 yunion.io/x/ovsdb/cli_util
 yunion.io/x/ovsdb/schema/ovn_nb
 yunion.io/x/ovsdb/types
-# yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
+# yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc
 yunion.io/x/pkg/errors
 yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/prettytable
@@ -1109,7 +1109,7 @@ yunion.io/x/pkg/util/workqueue
 yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v0.0.0-20201014101037-8fe75542e6d8
+# yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b
 yunion.io/x/sqlchemy
 # yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 yunion.io/x/structarg

vendor/yunion.io/x/pkg/util/compare/compare.go

@@ -97,8 +97,10 @@ func CompareSets(dbSet interface{}, extSet interface{}, removed interface{}, com
 				i += 1
 				j += 1
 			} else if cmp < 0 {
-				newVal := reflect.Append(removedValue, dbSetArray[i].value)
-				removedValue.Set(newVal)
+				if len(dbSetArray[i].key) > 0 {
+					newVal := reflect.Append(removedValue, dbSetArray[i].value)
+					removedValue.Set(newVal)
+				}
 				i += 1
 			} else {
 				newVal := reflect.Append(addedValue, extSetArray[j].value)
@@ -110,8 +112,10 @@ func CompareSets(dbSet interface{}, extSet interface{}, removed interface{}, com
 			addedValue.Set(newVal)
 			j += 1
 		} else if j >= len(extSetArray) {
-			newVal := reflect.Append(removedValue, dbSetArray[i].value)
-			removedValue.Set(newVal)
+			if len(dbSetArray[i].key) > 0 {
+				newVal := reflect.Append(removedValue, dbSetArray[i].value)
+				removedValue.Set(newVal)
+			}
 			i += 1
 		}
 	}

vendor/yunion.io/x/pkg/util/netutils/netutils.go

@@ -416,23 +416,14 @@ const (
 	multicastPrefix = "224.0.0.0/4"
 )
 
-var privatePrefixes = []string{
-	"10.0.0.0/8",
-	"172.16.0.0/12",
-	"192.168.0.0/16",
-}
-
 var privateIPRanges []IPV4AddrRange
 var hostLocalIPRange IPV4AddrRange
 var linkLocalIPRange IPV4AddrRange
 var multicastIPRange IPV4AddrRange
 
 func init() {
-	privateIPRanges = make([]IPV4AddrRange, len(privatePrefixes))
-	for i, prefix := range privatePrefixes {
-		prefix, _ := NewIPV4Prefix(prefix)
-		privateIPRanges[i] = prefix.ToIPRange()
-	}
+	updatePrivateIPRanges(nil)
+
 	prefix, _ := NewIPV4Prefix(hostlocalPrefix)
 	hostLocalIPRange = prefix.ToIPRange()
 	prefix, _ = NewIPV4Prefix(linklocalPrefix)
@@ -441,6 +432,32 @@ func init() {
 	multicastIPRange = prefix.ToIPRange()
 }
 
+func updatePrivateIPRanges(prefs []string) {
+	if len(prefs) == 0 {
+		prefs = []string{
+			"10.0.0.0/8",
+			"172.16.0.0/12",
+			"192.168.0.0/16",
+		}
+	}
+	privateIPRanges = make([]IPV4AddrRange, len(prefs))
+	for i, prefix := range prefs {
+		prefix, err := NewIPV4Prefix(prefix)
+		if err != nil {
+			continue
+		}
+		privateIPRanges[i] = prefix.ToIPRange()
+	}
+}
+
+func SetPrivatePrefixes(pref []string) {
+	updatePrivateIPRanges(pref)
+}
+
+func GetPrivateIPRanges() []IPV4AddrRange {
+	return privateIPRanges
+}
+
 func IsPrivate(addr IPV4Addr) bool {
 	for _, ipRange := range privateIPRanges {
 		if ipRange.Contains(addr) {

vendor/yunion.io/x/sqlchemy/query.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"fmt"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/log"
@@ -153,6 +154,10 @@ func (sq *SSubQuery) Expression() string {
 	for k := range sq.referedFields {
 		fields = append(fields, sq.referedFields[k])
 	}
+	// Make sure the order of the fields
+	sort.Slice(fields, func(i, j int) bool {
+		return fields[i].Name() < fields[j].Name()
+	})
 	return fmt.Sprintf("(%s)", sq.query.String(fields...))
 }