Merge pull request #8537 from rainzm/automated-cherry-pick-of-#8535-upstream-release-3.4

Automated cherry pick of #8535: refactor(region): faster list for cachedimage

pkg/apis/compute/guests.go

@@ -98,6 +98,9 @@ type ServerListInput struct {
 	SrcMacCheck *bool `json:"src_mac_check"`
 
 	InstanceType []string `json:"instance_type"`
+
+	// 是否调度到宿主机上
+	WithHost *bool `json:"with_host"`
 }
 
 func (input *ServerListInput) AfterUnmarshal() {

pkg/apis/compute/snapshotpolicy.go

@@ -21,10 +21,10 @@ type SnapshotPolicyDetails struct {
 
 	SSnapshotPolicy
 
-	RetentionDays  int   `json:"retention_days"`
-	RepeatWeekdays []int `json:"repeat_weekdays"`
-	TimePoints     []int `json:"time_points"`
-	IsActivated    *bool `json:"is_activated,omitempty"`
+	RetentionDays         int   `json:"retention_days"`
+	RepeatWeekdaysDisplay []int `json:"repeat_weekdays_display"`
+	TimePointsDisplay     []int `json:"time_points_display"`
+	IsActivated           *bool `json:"is_activated,omitempty"`
 
 	BindingDiskCount int `json:"binding_disk_count"`
 }

pkg/apis/monitor/commalert.go

@@ -86,6 +86,8 @@ type CommonAlertUpdateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// systemalert policy may need update through operator
+	ForceUpdate bool `json:"force_update"`
 }
 
 type CommonAlertDetails struct {

pkg/cloudcommon/db/caller.go

@@ -301,9 +301,9 @@ func FetchCustomizeColumns(
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
-		objDict := jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict)
-		objDict.Update(jsonDict)
-		retVal[i] = objDict
+		// NOTE: don't use obj update jsonDict as retval
+		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
+		retVal[i] = jsonDict
 	}
 	return retVal, nil
 }

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/notifyclient/notify.go

@@ -20,6 +20,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -36,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
@@ -159,7 +161,16 @@ func RawNotify(recipientId []string, isGroup bool, channel npk.TNotifyChannel, p
 	rawNotify(context.Background(), recipientId, isGroup, channel, priority, event, data)
 }
 
-func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+// IntelliNotify try to create receiver nonexistent if createReceiver is set to true
+func IntelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, createReceiver)
+}
+
+const noSuchReceiver = `no such receiver whose uid is '(.*)'`
+
+var noSuchReceiverRegexp = regexp.MustCompile(noSuchReceiver)
+
+func intelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
 	log.Infof("notify %s event %s priority %s", recipientId, event, priority)
 	msg := npk.SNotifyMessage{}
 	if isGroup {
@@ -182,13 +193,46 @@ func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel
 	// log.Debugf("send notification %s %s", topic, body)
 	notifyClientWorkerMan.Run(func() {
 		s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "")
-		err := npk.Notifications.Send(s, msg)
-		if err != nil {
-			log.Errorf("unable to send notification: %v", err)
+		for {
+			err := npk.Notifications.Send(s, msg)
+			if err == nil {
+				break
+			}
+			if !createReceiver {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			jerr, ok := err.(*httputils.JSONClientError)
+			if !ok {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			if jerr.Code > 500 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			match := noSuchReceiverRegexp.FindStringSubmatch(jerr.Details)
+			if match == nil || len(match) <= 1 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			receiverId := match[1]
+			createData := jsonutils.NewDict()
+			createData.Set("uid", jsonutils.NewString(receiverId))
+			_, err = modules.NotifyReceiver.Create(s, createData)
+			if err != nil {
+				log.Errorf("try to create receiver %q, but failed: %v", receiverId, err)
+				break
+			}
+			log.Infof("create receiver %q successfully", receiverId)
 		}
 	}, nil, nil)
 }
 
+func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, false)
+}
+
 func NotifyNormal(recipientId []string, isGroup bool, event string, data jsonutils.JSONObject) {
 	notifyNormal(context.Background(), recipientId, isGroup, event, data)
 }

pkg/compute/models/cachedimages.go

@@ -684,20 +684,6 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
-		cachedImages := CachedimageManager.Query().SubQuery()
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-
-		subq := cachedImages.Query(cachedImages.Field("id"))
-		subq = subq.Join(storagecachedImages, sqlchemy.Equals(cachedImages.Field("id"), storagecachedImages.Field("cachedimage_id")))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
-	}
-
 	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
@@ -708,74 +694,74 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByRegion(q, query.RegionalFilterListInput, "id", func() *sqlchemy.SQuery {
+	{
+		var idFilter bool
 		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
 		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
+		var storages *sqlchemy.SSubQuery
+
+		if query.Valid == nil {
+			storages = StorageManager.Query().SubQuery()
+		} else if *query.Valid {
+			idFilter = true
+			storages = StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
+		} else {
+			idFilter = true
+			stroage := StorageManager.Query()
+			storages = stroage.Filter(sqlchemy.OR(sqlchemy.NotIn(stroage.Field("status"), []string{}), sqlchemy.IsFalse(stroage.Field("enabled")))).SubQuery()
+		}
 		zones := ZoneManager.Query().SubQuery()
 
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByRegion")
-	}
 
-	q, err = managedResourceFilterByZone(q, query.ZonalFilterListInput, "id", func() *sqlchemy.SQuery {
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
-
-		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
+		if len(query.HostSchedtagId) > 0 {
+			idFilter = true
+			schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
+			if err != nil {
+				if errors.Cause(err) == sql.ErrNoRows {
+					return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
+				} else {
+					return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
+				}
+			}
+			hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
+			hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
+			subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
+			subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
+		}
 		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByZone")
+
+		subq = subq.Snapshot()
+
+		subq, err = managedResourceFilterByAccount(subq, query.ManagedResourceListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "managedResourceFilterByAccount")
+		}
+
+		subq, err = managedResourceFilterByRegion(subq, query.RegionalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByRegion")
+		}
+
+		subq, err = managedResourceFilterByZone(subq, query.ZonalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByZone")
+		}
+
+		if subq.IsAltered() {
+			idFilter = true
+		}
+
+		if idFilter {
+			q = q.In("id", subq)
+		}
 	}
 
 	if len(query.ImageType) > 0 {
-		q = q.In("image_type", query.ImageType)
-	}
-
-	if len(query.HostSchedtagId) > 0 {
-		schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
-		if err != nil {
-			if errors.Cause(err) == sql.ErrNoRows {
-				return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
-			} else {
-				return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
-			}
-		}
-		subq := StoragecachedimageManager.Query("cachedimage_id")
-		storages := StorageManager.Query("id", "storagecache_id").SubQuery()
-		hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
-		hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
-		subq = subq.Join(storages, sqlchemy.Equals(storages.Field("storagecache_id"), subq.Field("storagecache_id")))
-		subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
-		subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
-		q = q.In("id", subq.SubQuery())
-	}
-
-	if query.Valid != nil {
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
-
-		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
-		if *query.Valid {
-			q = q.In("id", subq.SubQuery())
-		} else {
-			q = q.NotIn("id", subq.SubQuery())
-		}
+		q = q.Equals("image_type", query.ImageType)
 	}
 
 	return q, nil

pkg/compute/models/cloudregions.go

@@ -118,6 +118,22 @@ func (self *SCloudregion) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudregion) GetElasticIps(managerId, eipMode string) ([]SElasticip, error) {
+	q := ElasticipManager.Query().Equals("cloudregion_id", self.Id)
+	if len(managerId) > 0 {
+		q = q.Equals("manager_id", managerId)
+	}
+	if len(eipMode) > 0 {
+		q = q.Equals("mode", eipMode)
+	}
+	eips := []SElasticip{}
+	err := db.FetchModelObjects(ElasticipManager, q, &eips)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return eips, nil
+}
+
 func (self *SCloudregion) GetZoneQuery() *sqlchemy.SQuery {
 	zones := ZoneManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {
@@ -464,6 +480,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()
 		self.Environment = cloudRegion.GetCloudEnv()
+		self.SetEnabled(true)
 
 		self.IsEmulated = cloudRegion.IsEmulated()
 

pkg/compute/models/dbinstances.go

@@ -1536,7 +1536,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extInstance.GetBillingType()
-			self.ExpiredAt = extInstance.GetExpiredAt()
+			if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extInstance.IsAutoRenew()
 		}
 
@@ -1626,7 +1628,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/disks.go

@@ -1543,7 +1543,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		disk.BillingType = extDisk.GetBillingType()
-		disk.ExpiredAt = extDisk.GetExpiredAt()
+		if expired := extDisk.GetExpiredAt(); !expired.IsZero() {
+			disk.ExpiredAt = expired
+		}
 		disk.AutoRenew = extDisk.IsAutoRenew()
 	}
 
@@ -2455,6 +2457,12 @@ func (self *SDisk) UpdataSnapshotsBackingDisk(backingDiskId string) error {
 func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 
 	now := time.Now()
+	week := now.Weekday()
+	if week == 0 {
+		week += 7
+	}
+	timePoint := now.Hour()
+
 	q := SnapshotPolicyDiskManager.Query().LE("next_sync_time", now)
 	spds := make([]SSnapshotPolicyDisk, 0)
 	err := db.FetchModelObjects(SnapshotPolicyDiskManager, q, &spds)
@@ -2484,7 +2492,11 @@ func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCr
 			db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT_FAIL, syncResult.Result(), userCred)
 			continue
 		}
-		if syncResult.AddCnt == 0 {
+		sp := spMap[spd.GetId()]
+		repeatWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+		timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+		if isInInts(int(week), repeatWeekdays) && isInInts(timePoint, timePoints) && syncResult.AddCnt == 0 {
+			// should add one
 			continue
 		}
 		db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT, "disk auto sync snapshot successfully", userCred)
@@ -2498,6 +2510,15 @@ func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCr
 	}
 }
 
+func isInInts(a int, array []int) bool {
+	for _, i := range array {
+		if i == a {
+			return true
+		}
+	}
+	return false
+}
+
 func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {
 	syncResult := compare.SyncResult{}
 

pkg/compute/models/elasticcache_instances.go

@@ -649,7 +649,9 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/elasticips.go

@@ -248,19 +248,6 @@ func (manager *SElasticipManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi
 	return q, httperrors.ErrNotFound
 }
 
-func (manager *SElasticipManager) getEipsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SElasticip, error) {
-	eips := make([]SElasticip, 0)
-	q := manager.Query().Equals("cloudregion_id", region.Id)
-	if provider != nil {
-		q = q.Equals("manager_id", provider.Id)
-	}
-	err := db.FetchModelObjects(manager, q, &eips)
-	if err != nil {
-		return nil, err
-	}
-	return eips, nil
-}
-
 func (self *SElasticip) GetRegion() *SCloudregion {
 	return CloudregionManager.FetchRegionById(self.CloudregionId)
 }
@@ -329,7 +316,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 	// remoteEips := make([]cloudprovider.ICloudEIP, 0)
 	syncResult := compare.SyncResult{}
 
-	dbEips, err := manager.getEipsByRegion(region, provider)
+	dbEips, err := region.GetElasticIps(provider.Id, api.EIP_MODE_STANDALONE_EIP)
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -478,7 +465,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.
 		factory, _ := provider.GetProviderFactory()
 		if factory != nil && factory.IsSupportPrepaidResources() {
 			self.BillingType = ext.GetBillingType()
-			self.ExpiredAt = ext.GetExpiredAt()
+			if expired := ext.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = ext.IsAutoRenew()
 		}
 

pkg/compute/models/guest_actions.go

@@ -3797,7 +3797,7 @@ func (self *SGuest) AllowPerformStreamDisksComplete(ctx context.Context, userCre
 func (self *SGuest) PerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	for _, disk := range self.GetDisks() {
 		d := disk.GetDisk()
-		if len(d.SnapshotId) > 0 {
+		if len(d.SnapshotId) > 0 && d.GetMetadata("merge_snapshot", userCred) == "true" {
 			SnapshotManager.AddRefCount(d.SnapshotId, -1)
 			d.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONFalse, userCred)
 		}

pkg/compute/models/guests.go

@@ -494,6 +494,13 @@ func (manager *SGuestManager) ListItemFilter(
 	if len(query.InstanceType) > 0 {
 		q = q.In("instance_type", query.InstanceType)
 	}
+	if query.WithHost != nil {
+		if *query.WithHost {
+			q = q.IsNotEmpty("host_id")
+		} else {
+			q = q.IsNullOrEmpty("host_id")
+		}
+	}
 
 	return q, nil
 }
@@ -2586,7 +2593,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		guest.BillingType = extVM.GetBillingType()
-		guest.ExpiredAt = extVM.GetExpiredAt()
+		if expired := extVM.GetExpiredAt(); !expired.IsZero() {
+			guest.ExpiredAt = expired
+		}
 		if guest.GetDriver().IsSupportSetAutoRenew() {
 			guest.AutoRenew = extVM.IsAutoRenew()
 		}
@@ -3121,12 +3130,11 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred
 
 func filterGuestByRange(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery {
 	hosts := HostManager.Query().SubQuery()
+	subq := hosts.Query(hosts.Field("id"))
+	subq = AttachUsageQuery(subq, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 
-	q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), q.Field("host_id")))
-	//q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-	// q = q.Filter(sqlchemy.Equals(hosts.Field("host_status"), HOST_ONLINE))
+	q = q.In("host_id", subq.SubQuery())
 
-	q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 	return q
 }
 
@@ -3234,7 +3242,9 @@ func _guestResourceCountQuery(
 
 	q = q.LeftJoin(isoDevSubQuery, sqlchemy.Equals(isoDevSubQuery.Field("guest_id"), guests.Field("id")))
 
-	q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	if len(rangeObjs) > 0 || len(hostTypes) > 0 || len(resourceTypes) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	}
 
 	switch scope {
 	case rbacutils.ScopeSystem:

pkg/compute/models/natgateways.go

@@ -363,7 +363,9 @@ func (self *SNatGateway) SyncWithCloudNatGateway(ctx context.Context, userCred m
 		factory, _ := provider.GetProviderFactory()
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extNat.GetBillingType()
-			self.ExpiredAt = extNat.GetExpiredAt()
+			if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extNat.IsAutoRenew()
 		}
 
@@ -407,7 +409,9 @@ func (manager *SNatGatewayManager) newFromCloudNatGateway(ctx context.Context, u
 	factory, _ := provider.GetProviderFactory()
 	if factory.IsSupportPrepaidResources() {
 		nat.BillingType = extNat.GetBillingType()
-		nat.ExpiredAt = extNat.GetExpiredAt()
+		if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+			nat.ExpiredAt = expired
+		}
 		nat.AutoRenew = extNat.IsAutoRenew()
 	}
 

pkg/compute/models/secgroups.go

@@ -943,6 +943,7 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, err
 	}
 
+	secgroup.Status = api.SECGROUP_STATUS_READY
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId

pkg/compute/models/snapshotpolicy.go

@@ -370,8 +370,8 @@ func (sp *SSnapshotPolicy) GetExtraDetails(
 }
 
 func (sp *SSnapshotPolicy) getMoreDetails(out api.SnapshotPolicyDetails) api.SnapshotPolicyDetails {
-	out.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
-	out.TimePoints = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+	out.RepeatWeekdaysDisplay = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+	out.TimePointsDisplay = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
 	out.BindingDiskCount, _ = SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	return out
 }

pkg/compute/models/snapshotpolicydisks.go

@@ -494,6 +494,16 @@ func (self *SSnapshotPolicyDiskManager) ValidateCreateData(ctx context.Context,
 	return data, nil
 }
 
+func (sd *SSnapshotPolicyDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	sp, err := SnapshotPolicyManager.FetchSnapshotPolicyById(sd.SnapshotpolicyId)
+	if err != nil {
+		return err
+	}
+	now := time.Now()
+	sd.NextSyncTime = sp.ComputeNextSyncTime(now, now)
+	return nil
+}
+
 func (sd *SSnapshotPolicyDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.
 	IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 

pkg/compute/models/snapshots.go

@@ -621,6 +621,11 @@ func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context) error {
 	if count > 0 {
 		return httperrors.NewBadRequestError("snapshot referenced by instance snapshot")
 	}
+	if disk, err := self.GetDisk(); err == nil {
+		if disk.Status == api.DISK_RESET {
+			return httperrors.NewBadRequestError("Cannot delete snapshot on disk reset")
+		}
+	}
 	driver := self.GetRegionDriver()
 	if driver != nil {
 		return driver.ValidateSnapshotDelete(ctx, self)

pkg/compute/usages/handler.go

@@ -676,7 +676,7 @@ func HostAllUsage(pref string, userCred mcclient.IIdentityProvider, scope rbacut
 	if len(pref) > 0 {
 		prefix = fmt.Sprintf("%s.%s", prefix, pref)
 	}
-	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.None)
+	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.None, tristate.False)
 }
 
 func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope, rangeObjs []db.IStandaloneModel,
@@ -685,7 +685,7 @@ func HostEnabledUsage(pref string, userCred mcclient.IIdentityProvider, scope rb
 	if len(pref) > 0 {
 		prefix = fmt.Sprintf("%s.%s", prefix, pref)
 	}
-	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.None)
+	return hostUsage(userCred, scope, prefix, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv, tristate.True, tristate.False)
 }
 
 func BaremetalUsage(userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope, rangeObjs []db.IStandaloneModel,

pkg/esxi/handler/handlers.go

@@ -75,7 +75,7 @@ func uploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		httperrors.MissingParameterError(ctx, w, "miss disk")
 		return
 	}
-	hostutils.DelayTask(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
+	hostutils.DelayTaskWithoutReqctx(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
 	hostutils.ResponseOk(ctx, w)
 }
 

pkg/hostman/guestman/guestman.go

@@ -291,6 +291,11 @@ func (m *SGuestManager) LoadServer(sid string) {
 		log.Errorf("On load server error: %s", err)
 		return
 	}
+
+	if jsonutils.QueryBoolean(guest.Desc, "need_sync_stream_disks", false) {
+		go guest.sendStreamDisksComplete(context.Background())
+	}
+
 	m.CandidateServers[sid] = guest
 }
 

pkg/hostman/guestman/qemu-kvm.go

@@ -1271,6 +1271,7 @@ func (s *SKVMGuestInstance) streamDisksComplete(ctx context.Context) {
 		if jsonutils.QueryBoolean(disk, "merge_snapshot", false) {
 			d := disks[i].(*jsonutils.JSONDict)
 			d.Set("merge_snapshot", jsonutils.JSONFalse)
+			s.Desc.Set("need_sync_stream_disks", jsonutils.JSONTrue)
 		}
 	}
 	if err := s.SaveDesc(s.Desc); err != nil {
@@ -1279,10 +1280,24 @@ func (s *SKVMGuestInstance) streamDisksComplete(ctx context.Context) {
 	if err := s.delFlatFiles(ctx); err != nil {
 		log.Errorf("del flat files failed %s", err)
 	}
-	_, err := modules.Servers.PerformAction(hostutils.GetComputeSession(ctx),
-		s.Id, "stream-disks-complete", nil)
-	if err != nil {
-		log.Infof("stream disks complete sync error %s", err)
+	go s.sendStreamDisksComplete(ctx)
+}
+
+func (s *SKVMGuestInstance) sendStreamDisksComplete(ctx context.Context) {
+	for {
+		_, err := modules.Servers.PerformAction(hostutils.GetComputeSession(ctx),
+			s.Id, "stream-disks-complete", nil)
+		if err != nil {
+			log.Errorf("stream disks complete sync error %s", err)
+			time.Sleep(30 * time.Second)
+			continue
+		} else {
+			break
+		}
+	}
+	s.Desc.Remove("need_sync_stream_disks")
+	if err := s.SaveDesc(s.Desc); err != nil {
+		log.Errorf("save guest desc failed %s", err)
 	}
 }
 

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -464,10 +464,10 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 	cmd += fmt.Sprintf(" -machine %s,accel=%s", s.getMachine(), accel)
 	cmd += " -k en-us"
 	// #cmd += " -g 800x600"
-	cmd += fmt.Sprintf(" -smp %d,maxcpus=128", cpu)
+	cmd += fmt.Sprintf(" -smp %d,maxcpus=255", cpu)
 	cmd += fmt.Sprintf(" -name %s", name)
 	// #cmd += fmt.Sprintf(" -uuid %s", self.desc["uuid"])
-	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=262144M", mem)
+	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=524288M", mem)
 
 	if s.manager.host.IsHugepagesEnabled() {
 		cmd += fmt.Sprintf(" -mem-prealloc -mem-path %s", fmt.Sprintf("/dev/hugepages/%s", uuid))

pkg/hostman/hostinfo/hostconsts/hostconsts.go

@@ -1,9 +1,11 @@
 package hostconsts
 
 const (
-	TELEGRAF_TAG_KEY_BRAND     = "brand"
-	TELEGRAF_TAG_KEY_RES_TYPE  = "res_type"
-	TELEGRAF_TAG_KEY_HOST_TYPE = "host_type"
+	TELEGRAF_TAG_KEY_BRAND      = "brand"
+	TELEGRAF_TAG_KEY_PLATFORM   = "platform"
+	TELEGRAF_TAG_KEY_HYPERVISOR = "hypervisor"
+	TELEGRAF_TAG_KEY_RES_TYPE   = "res_type"
+	TELEGRAF_TAG_KEY_HOST_TYPE  = "host_type"
 
 	TELEGRAF_TAG_ONECLOUD_BRAND                = "OneCloud"
 	TELEGRAF_TAG_ONECLOUD_RES_TYPE             = "host"

pkg/hostman/hostinfo/hostinfo.go

@@ -1108,8 +1108,10 @@ func (h *SHostInfo) PutHostOffline() {
 }
 
 func (h *SHostInfo) PutHostOnline() error {
-	if len(h.SysError) > 0 {
+	if len(h.SysError) > 0 && !options.HostOptions.StartHostIgnoreSysError {
 		log.Fatalf("Can't put host online, unless resolve these problem %v", h.SysError)
+	} else if len(h.SysError) > 0 && options.HostOptions.StartHostIgnoreSysError {
+		log.Errorf("Host sys error: %v", h.SysError)
 	}
 
 	if len(h.SysWarning) > 0 {

pkg/hostman/options/options.go

@@ -134,7 +134,8 @@ type SHostOptions struct {
 	HostHealthTimeout   int    `help:"host health timeout" default:"30"`
 	HostLeaseTimeout    int    `help:"lease timeout" default:"10"`
 
-	SyncStorageInfoDurationSecond int `help:"sync storage size duration, unit is second" default:"60"`
+	SyncStorageInfoDurationSecond int  `help:"sync storage size duration, unit is second" default:"60"`
+	StartHostIgnoreSysError       bool `help:"start host agent ignore sys error" default:"false"`
 }
 
 var (

pkg/image/models/image_guest.go

@@ -541,7 +541,7 @@ func (self *SGuestImage) PerformPublic(
 		return nil, errors.Wrap(err, "fail to fetch subimages of guest image")
 	}
 	for i := range images {
-		_, err := images[i].PerformPublic(ctx, userCred, query, input)
+		_, err := images[i].performPublic(ctx, userCred, query, input)
 		if err != nil {
 			return nil, errors.Wrapf(err, "fail to public subimage %s", images[i].GetId())
 		}
@@ -560,7 +560,7 @@ func (self *SGuestImage) PerformPrivate(
 		return nil, errors.Wrap(err, "fail to fetch subimages of guest image")
 	}
 	for i := range images {
-		_, err := images[i].PerformPrivate(ctx, userCred, query, input)
+		_, err := images[i].performPrivate(ctx, userCred, query, input)
 		if err != nil {
 			return nil, errors.Wrapf(err, "fail to private subimage %s", images[i].GetId())
 		}

pkg/image/models/images.go

@@ -1476,21 +1476,29 @@ func (img *SImage) PerformUpdateStatus(ctx context.Context, userCred mcclient.To
 }
 
 func (img *SImage) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) {
-	if img.IsStandard.IsTrue() {
-		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for standard image")
-	}
 	if img.IsGuestImage.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for guest image")
 	}
+	return img.performPublic(ctx, userCred, query, input)
+}
+
+func (img *SImage) performPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) {
+	if img.IsStandard.IsTrue() {
+		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for standard image")
+	}
 	return img.SSharableVirtualResourceBase.PerformPublic(ctx, userCred, query, input)
 }
 
-func (img *SImage) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
+func (img *SImage) performPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	if img.IsStandard.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform private for standard image")
 	}
+	return img.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input)
+}
+
+func (img *SImage) PerformPrivate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPrivateInput) (jsonutils.JSONObject, error) {
 	if img.IsGuestImage.IsTrue() {
 		return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform private for guest image")
 	}
-	return img.SSharableVirtualResourceBase.PerformPrivate(ctx, userCred, query, input)
+	return img.performPrivate(ctx, userCred, query, input)
 }

pkg/keystone/models/endpoints.go

@@ -20,6 +20,8 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"sync"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -47,6 +49,7 @@ type SEndpointManager struct {
 	SRegionResourceBaseManager
 
 	informerBackends map[string]informer.IInformerBackend
+	informerSetter   sync.Once
 }
 
 var EndpointManager *SEndpointManager
@@ -131,7 +134,8 @@ func (manager *SEndpointManager) SetInformerBackend() error {
 	if err != nil {
 		return errors.Wrap(err, "fetch informer endpoint")
 	}
-	return manager.SetInformerBackendByEndpoint(informerEp)
+	manager.SetInformerBackendUntilSuccess(informerEp)
+	return nil
 }
 
 func (manager *SEndpointManager) getSessionEndpointType() string {
@@ -160,11 +164,30 @@ func (manager *SEndpointManager) IsEtcdInformerBackend(ep *SEndpoint) bool {
 	return true
 }
 
-func (manager *SEndpointManager) SetInformerBackendByEndpoint(ep *SEndpoint) error {
-	if !manager.IsEtcdInformerBackend(ep) {
-		return nil
+func (manager *SEndpointManager) SetInformerBackendUntilSuccess(ep *SEndpoint) {
+	if informer.GetDefaultBackend() != nil {
+		log.Infof("Informer backend has been setted")
+		return
 	}
-	return manager.SetEtcdInformerBackend(ep)
+	manager.informerSetter.Do(func() {
+		go func() {
+			for {
+				if err := manager.SetEtcdInformerBackend(ep); err != nil {
+					log.Errorf("Set etcd informer backend failed: %s", err)
+					time.Sleep(time.Second * 30)
+				} else {
+					break
+				}
+			}
+		}()
+	})
+}
+
+func (manager *SEndpointManager) SetInformerBackendByEndpoint(ep *SEndpoint) {
+	if !manager.IsEtcdInformerBackend(ep) {
+		return
+	}
+	manager.SetInformerBackendUntilSuccess(ep)
 }
 
 func (manager *SEndpointManager) fetchInformerEndpoint() (*SEndpoint, error) {
@@ -587,9 +610,7 @@ func (manager *SEndpointManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fie
 }
 
 func (endpoint *SEndpoint) trySetInformerBackend() {
-	if err := EndpointManager.SetInformerBackendByEndpoint(endpoint); err != nil {
-		log.Errorf("Set informer by endpoint %s error: %v", endpoint.GetName(), err)
-	}
+	EndpointManager.SetInformerBackendByEndpoint(endpoint)
 }
 
 func (endpoint *SEndpoint) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {

pkg/mcclient/options/servers.go

@@ -71,6 +71,8 @@ type ServerListOptions struct {
 	WithMeta *bool `help:"filter by metadata" negative:"without_meta"`
 
 	WithUserMeta *bool `help:"filter by user metadata" negative:"without_user_meta"`
+
+	WithHost *bool `help:"filter guest with host or not" negative:"without_host"`
 }
 
 func (o *ServerListOptions) Params() (jsonutils.JSONObject, error) {

pkg/monitor/alerting/notifier.go

@@ -221,5 +221,9 @@ func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule {
 		alertRule.MeasurementDesc = evalCtx.EvalMatches[0].MeasurementDesc
 		alertRule.FieldDesc = evalCtx.EvalMatches[0].FieldDesc
 	}
+	if len(evalCtx.AlertOkEvalMatches) != 0 {
+		alertRule.MeasurementDesc = evalCtx.AlertOkEvalMatches[0].MeasurementDesc
+		alertRule.FieldDesc = evalCtx.AlertOkEvalMatches[0].FieldDesc
+	}
 	return alertRule
 }

pkg/monitor/alertresourcedrivers/cloudaccount.go

@@ -36,10 +36,10 @@ func (drvF cloudaccountDriverF) GetType() monitor.AlertResourceType {
 
 func (drvF cloudaccountDriverF) IsEvalMatched(input monitor.EvalMatch) bool {
 	tags := input.Tags
-	_, hasId := tags[CLOUDACCOUNT_TAG_ID_KEY]
-	if !hasId {
-		return false
-	}
+	//_, hasId := tags[CLOUDACCOUNT_TAG_ID_KEY]
+	//if !hasId {
+	//	return false
+	//}
 	_, hasName := tags[CLOUDACCOUNT_TAG_NAME_KEY]
 	if !hasName {
 		return false

pkg/monitor/models/alertrecord.go

@@ -171,6 +171,11 @@ func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.To
 		log.Errorf("Reconcile from alert record error: %v", err)
 		return
 	}
+	err := GetAlertResourceManager().NotifyAlertResourceCount(ctx)
+	if err != nil {
+		log.Errorf("NotifyAlertResourceCount error: %v", err)
+		return
+	}
 }
 
 func (record *SAlertRecord) GetState() monitor.AlertStateType {

pkg/monitor/models/alertresource.go

@@ -17,6 +17,7 @@ package models
 import (
 	"context"
 	"fmt"
+	"sync"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -27,11 +28,15 @@ import (
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	mc_modules "yunion.io/x/onecloud/pkg/mcclient/modules"
+	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 var (
 	alertResourceManager *SAlertResourceManager
+	adminUsers           *sync.Map
 )
 
 func init() {
@@ -373,3 +378,110 @@ func (res *SAlertResource) CustomizeDelete(
 	}
 	return nil
 }
+
+func (manager *SAlertResourceManager) NotifyAlertResourceCount(ctx context.Context) error {
+	log.Errorln("exec NotifyAlertResourceCount func")
+	cn, err := manager.getResourceCount()
+	if err != nil {
+		return err
+	}
+	alertResourceCount := resourceCount{
+		AlertResourceCount: cn,
+	}
+	if adminUsers == nil {
+		manager.GetAdminRoleUsers(ctx, nil, true)
+	}
+	adminUsersTmp := *adminUsers
+	ids := make([]string, 0)
+	adminUsersTmp.Range(func(key, value interface{}) bool {
+		ids = append(ids, key.(string))
+		return true
+	})
+	if len(ids) == 0 {
+		return fmt.Errorf("no find users in receivers has admin role")
+	}
+	//if len(ids) != 0 {
+	//	notifyclient.RawNotifyWithCtx(ctx, ids, false, npk.NotifyByWebConsole, npk.NotifyPriorityCritical,
+	//		"alertResourceCount", jsonutils.Marshal(&alertResourceCount))
+	//	return nil
+	//} else {
+	//	return fmt.Errorf("no find users in receivers has admin role")
+	//}
+	manager.sendWebsocketInfo(ids, alertResourceCount)
+	return nil
+}
+
+type resourceCount struct {
+	AlertResourceCount int `json:"alert_resource_count"`
+}
+
+func (manager *SAlertResourceManager) getResourceCount() (int, error) {
+	query := manager.Query("id")
+	cn, err := query.CountWithError()
+	if err != nil {
+		return cn, errors.Wrap(err, "SAlertResourceManager get resource count error")
+	}
+
+	return cn, nil
+}
+
+func (manager *SAlertResourceManager) GetAdminRoleUsers(ctx context.Context, userCred mcclient.TokenCredential,
+	isStart bool) {
+	if adminUsers == nil {
+		adminUsers = new(sync.Map)
+	}
+	offset := 0
+	query := jsonutils.NewDict()
+	session := auth.GetAdminSession(ctx, "", "")
+	rid, err := mc_modules.RolesV3.GetId(session, "admin", jsonutils.NewDict())
+	if err != nil {
+		errors.Errorf("get role id error:%v", err)
+		return
+	}
+	query.Add(jsonutils.NewString(rid), "role", "id")
+	for {
+		query.Set("offset", jsonutils.NewInt(int64(offset)))
+		result, err := mc_modules.RoleAssignments.List(session, query)
+		if err != nil {
+			errors.Errorf("get admin role list error:%v", err)
+			return
+		}
+		for _, roleAssign := range result.Data {
+			userId, err := roleAssign.GetString("user", "id")
+			if err != nil {
+				log.Errorf("roleAssign:%v", roleAssign)
+				return
+			}
+			//_, err = mc_modules.NotifyReceiver.GetById(session, userId, jsonutils.NewDict())
+			//if err != nil {
+			//	log.Errorf("Recipients GetById err:%v", err)
+			//	continue
+			//}
+			adminUsers.Store(userId, roleAssign)
+		}
+		offset = result.Offset + len(result.Data)
+		if offset >= result.Total {
+			break
+		}
+	}
+}
+
+func (manager *SAlertResourceManager) sendWebsocketInfo(uids []string, alertResourceCount resourceCount) {
+	session := auth.GetAdminSession(context.Background(), "", "")
+	params := jsonutils.NewDict()
+	params.Set("obj_type", jsonutils.NewString("monitor"))
+	params.Set("obj_id", jsonutils.NewString(""))
+	params.Set("obj_name", jsonutils.NewString(""))
+	params.Set("success", jsonutils.JSONTrue)
+	params.Set("action", jsonutils.NewString("alertResourceCount"))
+	params.Set("notes", jsonutils.NewString(fmt.Sprintf("priority=%s; content=%s", string(npk.NotifyPriorityCritical),
+		jsonutils.Marshal(&alertResourceCount).String())))
+	for _, uid := range uids {
+		params.Set("user_id", jsonutils.NewString(uid))
+		params.Set("user", jsonutils.NewString(uid))
+		_, err := mc_modules.Websockets.Create(session, params)
+		if err != nil {
+			log.Errorf("websocket send info err:%v", err)
+		}
+	}
+}

pkg/monitor/models/commonalert.go

@@ -706,9 +706,11 @@ func (alert *SCommonAlert) ValidateUpdateData(
 		if err != nil {
 			return data, errors.Wrap(err, "metric query error")
 		}
-
 		if alert.getAlertType() == monitor.CommonAlertSystemAlertType {
-			return data, nil
+			forceUpdate, _ := data.Bool("force_update")
+			if !forceUpdate {
+				return data, nil
+			}
 		}
 		data.Update(jsonutils.Marshal(metricQuery))
 		err = data.Unmarshal(updataInput)

pkg/monitor/models/datasource.go

@@ -34,9 +34,9 @@ import (
 	"yunion.io/x/pkg/util/wait"
 	"yunion.io/x/pkg/utils"
 
-	identityapi "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	merrors "yunion.io/x/onecloud/pkg/monitor/errors"
@@ -94,25 +94,32 @@ func (man *SDataSourceManager) Run(ctx context.Context) error {
 
 func (man *SDataSourceManager) initDefaultDataSource(ctx context.Context) error {
 	region := options.Options.Region
+	epType := options.Options.SessionEndpointType
 	initF := func() {
 		ds, err := man.GetDefaultSource()
 		if err != nil && err != ErrDataSourceDefaultNotFound {
 			log.Errorf("Get default datasource: %v", err)
 			return
 		}
-		if ds != nil {
-			return
-		}
-		s := auth.GetAdminSessionWithPublic(ctx, region, "")
+		s := auth.GetAdminSession(ctx, region, "")
 		if s == nil {
 			log.Errorf("get empty public session for region %s", region)
 			return
 		}
-		url, err := s.GetServiceURL("influxdb", identityapi.EndpointInterfacePublic)
+		url, err := s.GetServiceURL("influxdb", epType)
 		if err != nil {
 			log.Errorf("get influxdb public url: %v", err)
 			return
 		}
+		if ds != nil {
+			if _, err := db.Update(ds, func() error {
+				ds.Url = url
+				return nil
+			}); err != nil {
+				log.Errorf("update datasource url error: %v", err)
+			}
+			return
+		}
 		ds = &SDataSource{
 			Type: monitor.DataSourceTypeInfluxdb,
 			Url:  url,
@@ -608,10 +615,34 @@ func (self *SDataSourceManager) GetMetricMeasurement(query jsonutils.JSONObject,
 	if err != nil {
 		return jsonutils.JSONNull, errors.Wrap(err, "getTagValue error")
 	}
+	self.filterRtnTags(output)
 	return jsonutils.Marshal(output), nil
 
 }
 
+func (self *SDataSourceManager) filterRtnTags(output *monitor.InfluxMeasurement) {
+	for _, tag := range []string{hostconsts.TELEGRAF_TAG_KEY_BRAND, hostconsts.TELEGRAF_TAG_KEY_PLATFORM,
+		hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR} {
+		if val, ok := output.TagValue[tag]; ok {
+			output.TagValue[hostconsts.TELEGRAF_TAG_KEY_BRAND] = val
+			break
+		}
+	}
+	for _, tag := range []string{"source", "status", hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE,
+		hostconsts.TELEGRAF_TAG_KEY_RES_TYPE, "is_vm", "os_type", hostconsts.TELEGRAF_TAG_KEY_PLATFORM,
+		hostconsts.TELEGRAF_TAG_KEY_HYPERVISOR, "domain_name", "region"} {
+		if _, ok := output.TagValue[tag]; ok {
+			delete(output.TagValue, tag)
+		}
+	}
+
+	repTag := make([]string, 0)
+	for tag, _ := range output.TagValue {
+		repTag = append(repTag, tag)
+	}
+	output.TagKey = repTag
+}
+
 func (self *SDataSourceManager) filterTagValue(measurement monitor.InfluxMeasurement, timeF timeFilter,
 	db *influxdb.SInfluxdb, tagValChan *influxdbTagValueChan, tagFilter string) error {
 	ctx, _ := context.WithTimeout(context.Background(), time.Second*5)

pkg/monitor/models/unifiedmonitor.go

@@ -11,6 +11,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	merrors "yunion.io/x/onecloud/pkg/monitor/errors"
@@ -392,6 +393,12 @@ func fillSerieTags(series *tsdb.TimeSeriesSlice) {
 				break
 			}
 		}
+		for _, tag := range []string{"source", "status", hostconsts.TELEGRAF_TAG_KEY_HOST_TYPE,
+			hostconsts.TELEGRAF_TAG_KEY_RES_TYPE, "cpu", "is_vm", "os_type", "domain_name", "region"} {
+			if _, ok := serie.Tags[tag]; ok {
+				delete(serie.Tags, tag)
+			}
+		}
 		(*series)[i] = serie
 	}
 }

pkg/monitor/options/options.go

@@ -22,12 +22,13 @@ type AlerterOptions struct {
 	common_options.CommonOptions
 	common_options.DBOptions
 
-	DataProxyTimeout                      int   `help:"query data source proxy timeout" default:"30"`
-	AlertingMinIntervalSeconds            int64 `help:"alerting min schedule frequency" default:"10"`
-	AlertingMaxAttempts                   int   `help:"alerting engine max attempt" default:"3"`
-	AlertingEvaluationTimeoutSeconds      int64 `help:"alerting evaluation timeout" default:"5"`
-	AlertingNotificationTimeoutSeconds    int64 `help:"alerting notification timeout" default:"30"`
-	InitScopeSuggestConfigIntervalSeconds int   `help:"internal to init scope suggest configs" default:"900"`
+	DataProxyTimeout                               int   `help:"query data source proxy timeout" default:"30"`
+	AlertingMinIntervalSeconds                     int64 `help:"alerting min schedule frequency" default:"10"`
+	AlertingMaxAttempts                            int   `help:"alerting engine max attempt" default:"3"`
+	AlertingEvaluationTimeoutSeconds               int64 `help:"alerting evaluation timeout" default:"5"`
+	AlertingNotificationTimeoutSeconds             int64 `help:"alerting notification timeout" default:"30"`
+	InitScopeSuggestConfigIntervalSeconds          int   `help:"internal to init scope suggest configs" default:"900"`
+	InitAlertResourceAdminRoleUsersIntervalSeconds int   `help:"internal to init alert resource admin role users " default:"3600"`
 }
 
 var (

pkg/monitor/service/handlers.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"strconv"
 
 	"github.com/gorilla/mux"
@@ -120,4 +121,13 @@ func addMiscHandlers(root *mux.Router) {
 		}
 	}
 	root.HandleFunc("/subscriptions/write", adapterF(performHandler))
+
+	// ref: pkg/appsrv/appsrv:addDefaultHandlers
+	root.HandleFunc("/version", adapterF(appsrv.VersionHandler))
+	root.HandleFunc("/stats", adapterF(appsrv.StatisticHandler))
+	root.HandleFunc("/ping", adapterF(appsrv.PingHandler))
+	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
+
+	// pprof handler
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }

pkg/monitor/service/service.go

@@ -66,6 +66,7 @@ func StartService() {
 	cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount)
 	suggestsysdrivers.InitSuggestSysRuleCronjob()
 	cron.AddJobAtIntervalsWithStartRun("InitScopeSuggestConfigs", time.Duration(opts.InitScopeSuggestConfigIntervalSeconds)*time.Second, models.SuggestSysRuleConfigManager.InitScopeConfigs, true)
+	cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true)
 	cron.Start()
 	defer cron.Stop()
 

pkg/multicloud/aliyun/latitud_and_longitude.go

@@ -24,6 +24,7 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
 	"cn-beijing":            {Latitude: 39.904202, Longitude: 116.407394, City: api.CITY_BEI_JING, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-zhangjiakou":        {Latitude: 40.767544, Longitude: 114.886337, City: api.CITY_ZHANG_JIA_KOU, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-huhehaote":          {Latitude: 40.842358, Longitude: 111.749992, City: api.CITY_HU_HE_HAO_TE, CountryCode: api.COUNTRY_CODE_CN},
+	"cn-huhehaote-nebula-1": {Latitude: 40.842358, Longitude: 111.749992, City: api.CITY_HU_HE_HAO_TE, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-hangzhou":           {Latitude: 30.274084, Longitude: 120.155067, City: api.CITY_HANG_ZHOU, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-shanghai":           {Latitude: 31.230391, Longitude: 121.473701, City: api.CITY_SHANG_HAI, CountryCode: api.COUNTRY_CODE_CN},
 	"cn-shanghai-finance-1": {Latitude: 31.230391, Longitude: 121.473701, City: api.CITY_SHANG_HAI, CountryCode: api.COUNTRY_CODE_CN},
@@ -44,4 +45,5 @@ var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
 	"me-east-1":             {Latitude: 25.204849, Longitude: 55.270782, City: api.CITY_DUBAI, CountryCode: api.COUNTRY_CODE_AE},
 	"eu-central-1":          {Latitude: 50.110924, Longitude: 8.682127, City: api.CITY_FRANKFURT, CountryCode: api.COUNTRY_CODE_DE},
 	"cn-wulanchabu":         {Latitude: 41.0178065, Longitude: 113.094978, City: api.CITY_WU_LAN_CHA_BU, CountryCode: api.COUNTRY_CODE_CN},
+	"cn-guangzhou":          {Latitude: 23.129110, Longitude: 113.264381, City: api.CITY_GUANG_ZHOU, CountryCode: api.COUNTRY_CODE_CN},
 }

pkg/multicloud/esxi/host.go

@@ -966,6 +966,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 		}
 	}
 
+	var rootDiskSizeMb int64
 	if len(params.Disks) > 0 {
 		driver := params.Disks[0].Driver
 		if driver == "scsi" || driver == "pvscsi" {
@@ -993,30 +994,10 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 			}
 		}
 
-		// resize system disk
-		sysDiskSize := params.Disks[0].Size
-		if sysDiskSize == 0 {
-			sysDiskSize = 30 * 1024
+		rootDiskSizeMb = params.Disks[0].Size
+		if rootDiskSizeMb == 0 {
+			rootDiskSizeMb = 30 * 1024
 		}
-		if int64(from.vdisks[0].GetDiskSizeMB()) != sysDiskSize {
-			vdisk := from.vdisks[0].getVirtualDisk()
-			vdisk.CapacityInKB = sysDiskSize * 1024
-			spec := &types.VirtualDeviceConfigSpec{}
-			spec.Operation = types.VirtualDeviceConfigSpecOperationEdit
-			spec.Device = vdisk
-			deviceChange = append(deviceChange, spec)
-			log.Infof("resize system disk: %dGB => %dGB", from.vdisks[0].GetDiskSizeMB()/1024, vdisk.CapacityInKB/1024/1024)
-		}
-		// remove extra disk
-		// for i := 1; i < len(from.vdisks); i++ {
-		// 	 dev := from.vdisks[i].dev
-		// 	 spec := &types.VirtualDeviceConfigSpec{}
-		// 	 spec.Operation = types.VirtualDeviceConfigSpecOperationRemove
-		// 	 spec.Device = dev
-		//	 spec.FileOperation = types.VirtualDeviceConfigSpecFileOperationDestroy
-		//	 deviceChange = append(deviceChange, spec)
-		//	 log.Debugf("remove disk, index: %d", i)
-		// }
 	}
 
 	dc, err := host.GetDatacenter()
@@ -1081,6 +1062,13 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 	if vm == nil {
 		return nil, errors.Error("clone successfully but unable to NewVirtualMachine")
 	}
+	// resize system disk
+	if rootDiskSizeMb > 0 && int64(vm.vdisks[0].GetDiskSizeMB()) != rootDiskSizeMb {
+		err = vm.vdisks[0].Resize(ctx, rootDiskSizeMb)
+		if err != nil {
+			return vm, errors.Wrap(err, "resize for root disk")
+		}
+	}
 	// add data disk
 	for i := 1; i < len(params.Disks); i++ {
 		size := params.Disks[i].Size

pkg/multicloud/esxi/virtualmachine.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/vmware/govmomi/nfc"
 	"github.com/vmware/govmomi/object"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/soap"
@@ -730,8 +731,13 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		return fmt.Errorf("invalid vm")
 	}
 
-	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
-		dev := moVM.Config.Hardware.Device[i]
+	// sort devices via their Key
+	devices := moVM.Config.Hardware.Device
+	sort.Slice(devices, func(i, j int) bool {
+		return devices[i].GetVirtualDevice().Key < devices[j].GetVirtualDevice().Key
+	})
+	for i := 0; i < len(devices); i += 1 {
+		dev := devices[i]
 		devType := reflect.Indirect(reflect.ValueOf(dev)).Type()
 
 		etherType := reflect.TypeOf((*types.VirtualEthernetCard)(nil)).Elem()
@@ -742,7 +748,7 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 		if reflectutils.StructContains(devType, etherType) {
 			self.vnics = append(self.vnics, NewVirtualNIC(self, dev, len(self.vnics)))
 		} else if reflectutils.StructContains(devType, diskType) {
-			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vnics)))
+			self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vdisks)))
 		} else if reflectutils.StructContains(devType, vgaType) {
 			self.vga = NewVirtualVGA(self, dev, 0)
 		} else if reflectutils.StructContains(devType, cdromType) {
@@ -1175,8 +1181,19 @@ func (self *SVirtualMachine) ExportTemplate(ctx context.Context, idx int, diskPa
 	lr := newLeaseLogger("download vmdk", 5)
 	lr.Log()
 	defer lr.End()
+
+	// filter vmdk item
+	vmdkItems := make([]nfc.FileItem, 0, len(info.Items)/2)
+	for i := range info.Items {
+		if strings.HasSuffix(info.Items[i].Path, ".vmdk") {
+			vmdkItems = append(vmdkItems, info.Items[i])
+		} else {
+			log.Infof("item.Path does not end in '.vmdk': %#v", info.Items[i])
+		}
+	}
+
 	log.Debugf("download to %s start...", diskPath)
-	err = lease.DownloadFile(ctx, diskPath, info.Items[idx], soap.Download{Progress: lr})
+	err = lease.DownloadFile(ctx, diskPath, vmdkItems[idx], soap.Download{Progress: lr})
 	if err != nil {
 		return errors.Wrap(err, "lease.DownloadFile")
 	}

pkg/multicloud/qcloud/dnspod_domain.go

@@ -58,10 +58,14 @@ type SDomian struct {
 }
 
 // https://cloud.tencent.com/document/product/302/8505
-func (client *SQcloudClient) GetDomains(offset int, limit int) ([]SDomian, int, error) {
+func (client *SQcloudClient) GetDomains(projectId string, offset int, limit int) ([]SDomian, int, error) {
 	params := map[string]string{}
 	params["offset"] = strconv.Itoa(offset)
 	params["length"] = strconv.Itoa(limit)
+	if len(projectId) > 0 {
+		params["qProjectId"] = projectId
+	}
+
 	resp, err := client.cnsRequest("DomainList", params)
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "client.cnsRequest(DomainList, %s)", jsonutils.Marshal(params).String())
@@ -87,7 +91,8 @@ func (client *SQcloudClient) GetAllDomains() ([]SDomian, error) {
 	count := 0
 	result := []SDomian{}
 	for {
-		domains, total, err := client.GetDomains(count, 100)
+		// -1 所有项目; 0,default默认项目
+		domains, total, err := client.GetDomains("-1", count, 100)
 		if err != nil {
 			return nil, errors.Wrap(err, " client.GetDomains(count, 100)")
 		}

pkg/multicloud/qcloud/dnspod_record.go

@@ -57,12 +57,15 @@ type SDnsRecord struct {
 }
 
 // https://cloud.tencent.com/document/product/302/8517
-func (client *SQcloudClient) GetDnsRecords(sDomainName string, offset int, limit int) ([]SDnsRecord, int, error) {
+func (client *SQcloudClient) GetDnsRecords(projectId string, sDomainName string, offset int, limit int) ([]SDnsRecord, int, error) {
 
 	params := map[string]string{}
 	params["offset"] = strconv.Itoa(offset)
 	params["length"] = strconv.Itoa(limit)
 	params["domain"] = sDomainName
+	if len(projectId) > 0 {
+		params["qProjectId"] = projectId
+	}
 	resp, err := client.cnsRequest("RecordList", params)
 	if err != nil {
 		return nil, 0, errors.Wrapf(err, "client.cnsRequest(RecordList, %s)", jsonutils.Marshal(params).String())
@@ -88,7 +91,8 @@ func (client *SQcloudClient) GetAllDnsRecords(sDomainName string) ([]SDnsRecord,
 	count := 0
 	result := []SDnsRecord{}
 	for true {
-		records, total, err := client.GetDnsRecords(sDomainName, count, 100)
+		// -1 所有项目; 0,default默认项目
+		records, total, err := client.GetDnsRecords("-1", sDomainName, count, 100)
 		if err != nil {
 			return nil, errors.Wrapf(err, "client.GetDnsRecords(%s,%d,%d)", sDomainName, count, 100)
 		}

pkg/multicloud/qcloud/project.go

@@ -79,6 +79,9 @@ func (client *SQcloudClient) GetProjects() ([]SProject, error) {
 	projects := []SProject{}
 	params := map[string]string{"allList": "1"}
 	resp, err := client.accountRequestRequest("DescribeProject", params)
+	if err != nil {
+		return nil, errors.Wrapf(err, "DescribeProject")
+	}
 	err = resp.Unmarshal(&projects)
 	if err != nil {
 		return nil, errors.Wrap(err, "resp.Unmarshal")

pkg/multicloud/qcloud/shell/dnspod.go

@@ -24,11 +24,12 @@ import (
 
 func init() {
 	type DomianListOptions struct {
-		Offset int
-		Limit  int
+		Offset    int
+		Limit     int
+		ProjectId string
 	}
 	shellutils.R(&DomianListOptions{}, "domain-list", "List domains", func(cli *qcloud.SRegion, args *DomianListOptions) error {
-		domains, total, e := cli.GetClient().GetDomains(args.Offset, args.Limit)
+		domains, total, e := cli.GetClient().GetDomains(args.ProjectId, args.Offset, args.Limit)
 		if e != nil {
 			return e
 		}
@@ -61,12 +62,13 @@ func init() {
 	})
 
 	type DnsRecordListOptions struct {
-		DOMAIN string
-		Offset int
-		Limit  int
+		DOMAIN    string
+		ProjectId string
+		Offset    int
+		Limit     int
 	}
 	shellutils.R(&DnsRecordListOptions{}, "dnsrecord-list", "List dndrecord", func(cli *qcloud.SRegion, args *DnsRecordListOptions) error {
-		records, total, e := cli.GetClient().GetDnsRecords(args.DOMAIN, args.Offset, args.Limit)
+		records, total, e := cli.GetClient().GetDnsRecords(args.ProjectId, args.DOMAIN, args.Offset, args.Limit)
 		if e != nil {
 			return e
 		}
@@ -84,7 +86,7 @@ func init() {
 	}
 	shellutils.R(&DnsRecordCreateOptions{}, "dnsrecord-create", "create dndrecord", func(cli *qcloud.SRegion, args *DnsRecordCreateOptions) error {
 		change := cloudprovider.DnsRecordSet{}
-		change.DnsName = args.DOMAIN
+		change.DnsName = args.NAME
 		change.DnsValue = args.VALUE
 		change.Ttl = args.TTL
 		change.DnsType = cloudprovider.TDnsType(args.TYPE)

pkg/notify/models/receiver.go

@@ -63,6 +63,10 @@ var (
 		api.DINGTALK_ROBOT,
 		api.WORKWX_ROBOT,
 	}
+	AllOkContactTypes = append(AllRobotContactTypes,
+		api.WEBCONSOLE,
+		api.WEBHOOK,
+	)
 )
 
 type SReceiverManager struct {
@@ -264,18 +268,18 @@ func (rm *SReceiverManager) ValidateCreateData(ctx context.Context, userCred mcc
 	// hack
 	input.Name = input.UName
 	// validate email
-	if ok := regutils.MatchEmail(input.Email); !ok {
+	if ok := regutils.MatchEmail(input.Email); len(input.Email) > 0 && !ok {
 		return input, httperrors.NewInputParameterError("invalid email")
 	}
 	// validate mobile
-	if ok := regutils.MatchMobile(input.Mobile); !ok {
+	if ok := regutils.MatchMobile(input.Mobile); len(input.Mobile) > 0 && !ok {
 		return input, httperrors.NewInputParameterError("invalid mobile")
 	}
 	return input, nil
 }
 
 func (r *SReceiver) IsEnabledContactType(ct string) (bool, error) {
-	if utils.IsInStringArray(ct, AllRobotContactTypes) {
+	if utils.IsInStringArray(ct, AllOkContactTypes) {
 		return true, nil
 	}
 	if ct == api.WEBCONSOLE {
@@ -289,7 +293,7 @@ func (r *SReceiver) IsEnabledContactType(ct string) (bool, error) {
 }
 
 func (r *SReceiver) IsVerifiedContactType(ct string) (bool, error) {
-	if utils.IsInStringArray(ct, AllRobotContactTypes) {
+	if utils.IsInStringArray(ct, AllOkContactTypes) {
 		return true, nil
 	}
 	cts, err := r.GetVerifiedContactTypes()

pkg/webconsole/service/service.go

@@ -18,7 +18,7 @@ import (
 	"context"
 	"net"
 	"net/http"
-	"net/http/pprof"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"strconv"
@@ -133,9 +133,5 @@ func addMiscHandlers(root *mux.Router) {
 	root.HandleFunc("/worker_stats", adapterF(appsrv.WorkerStatsHandler))
 
 	// pprof handler
-	root.HandleFunc("/debug/pprof/", pprof.Index)
-	root.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-	root.HandleFunc("/debug/pprof/profile", pprof.Profile)
-	root.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-	root.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	root.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
 }