feat(monitor): alert add silent period param

1. 报警静默期支持

pkg/apis/monitor/alertrecord.go

@@ -2,6 +2,11 @@ package monitor
 
 import "yunion.io/x/onecloud/pkg/apis"
 
+const (
+	SEND_STATE_OK     = "ok"
+	SEND_STATE_SILENT = "silent"
+)
+
 type AlertRecordListInput struct {
 	apis.Meta
 
@@ -31,6 +36,7 @@ type AlertRecordCreateInput struct {
 	// 报警级别
 	Level     string       `json:"level"`
 	State     string       `json:"state"`
+	SendState string       `json:"send_state"`
 	ResType   string       `json:"res_type"`
 	EvalData  []*EvalMatch `json:"eval_data"`
 	AlertRule AlertRecordRule

pkg/apis/monitor/commalert.go

@@ -42,6 +42,8 @@ type CommonAlertCreateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// 静默期
+	SilentPeriod string `json:"silent_period"`
 	// 报警类型
 	AlertType string `json:"alert_type"`
 
@@ -98,6 +100,8 @@ type CommonAlertUpdateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// 静默期
+	SilentPeriod string `json:"silent_period"`
 	// systemalert policy may need update through operator
 	ForceUpdate bool   `json:"force_update"`
 	GetPointStr bool   `json:"get_point_str"`

pkg/apis/monitor/notification.go

@@ -50,7 +50,7 @@ type NotificationCreateInput struct {
 	SendReminder *bool `json:"send_reminder"`
 	// 是否禁用报警恢复提醒
 	DisableResolveMessage *bool `json:"disable_resolve_message"`
-	// 发送频率
+	// 发送频率 单位：s
 	Frequency time.Duration `json:"frequency"`
 	// 通知配置
 	Settings jsonutils.JSONObject `json:"settings"`

pkg/monitor/alerting/notifier.go

@@ -144,8 +144,8 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon
 			})
 		}
 	}
-	if shouldNotify {
-		n.createAlertRecordWhenNotify(evalCtx)
+	if shouldNotify || evalCtx.Rule.State == monitor.AlertStateAlerting {
+		n.createAlertRecordWhenNotify(evalCtx, shouldNotify)
 	}
 	if !shouldNotify && evalCtx.shouldUpdateAlertState() && evalCtx.NoDataFound {
 		n.detachAlertResourceWhenNodata(evalCtx)
@@ -154,7 +154,7 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon
 	return result, nil
 }
 
-func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext) {
+func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext, shouldNotify bool) {
 	var matches []*monitor.EvalMatch
 	if evalCtx.Firing {
 		matches = evalCtx.EvalMatches
@@ -168,9 +168,13 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext)
 		AlertId:   evalCtx.Rule.Id,
 		Level:     evalCtx.Rule.Level,
 		State:     string(evalCtx.Rule.State),
+		SendState: monitor.SEND_STATE_OK,
 		EvalData:  matches,
 		AlertRule: newAlertRecordRule(evalCtx),
 	}
+	if !shouldNotify {
+		recordCreateInput.SendState = monitor.SEND_STATE_SILENT
+	}
 	recordCreateInput.ResType = recordCreateInput.AlertRule.ResType
 	createData := recordCreateInput.JSON(recordCreateInput)
 	alert, _ := models.CommonAlertManager.GetAlert(evalCtx.Rule.Id)

pkg/monitor/alerting/notifiers/base.go

@@ -66,6 +66,9 @@ func (n *NotifierBase) ShouldNotify(_ context.Context, evalCtx *alerting.EvalCon
 	}
 
 	if newState == monitor.AlertStateAlerting {
+		if prevState == monitor.AlertStateOK {
+			return true
+		}
 		send, err := state.ShouldSendNotification()
 		if err != nil {
 			log.Errorf("Alertnotification ShouldSendNotification exec err:%v", err)

pkg/monitor/models/alertrecord.go

@@ -38,6 +38,7 @@ type SAlertRecord struct {
 	AlertId   string               `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"`
 	Level     string               `charset:"ascii" width:"36" nullable:"false" default:"normal" list:"user" update:"user"`
 	State     string               `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"`
+	SendState string               `width:"36" charset:"ascii" default:"ok" list:"user" update:"user"`
 	EvalData  jsonutils.JSONObject `list:"user" update:"user"`
 	AlertRule jsonutils.JSONObject `list:"user" update:"user"`
 	ResType   string               `width:"36" list:"user" update:"user"`

pkg/monitor/models/commonalert.go

@@ -113,6 +113,11 @@ func (man *SCommonAlertManager) ValidateCreateData(
 	if _, err := time.ParseDuration(data.Period); err != nil {
 		return data, httperrors.NewInputParameterError("Invalid period format: %s", data.Period)
 	}
+	if data.SilentPeriod != "" {
+		if _, err := time.ParseDuration(data.SilentPeriod); err != nil {
+			return data, httperrors.NewInputParameterError("Invalid silent_period format: %s", data.SilentPeriod)
+		}
+	}
 	// 默认的系统配置Recipients=commonalert-default
 	if data.AlertType != monitor.CommonAlertSystemAlertType && len(data.Recipients) == 0 {
 		return data, merrors.NewArgIsEmptyErr("recipients")
@@ -253,10 +258,10 @@ func (alert *SCommonAlert) customizeCreateNotis(ctx context.Context, userCred mc
 	}
 	//user_by 弃用
 	if input.AlertType == monitor.CommonAlertSystemAlertType {
-		return alert.createAlertNoti(ctx, userCred, input.Name, "webconsole", []string{}, true)
+		return alert.createAlertNoti(ctx, userCred, input.Name, "webconsole", []string{}, input.SilentPeriod, true)
 	}
 	for _, channel := range input.Channel {
-		err := alert.createAlertNoti(ctx, userCred, input.Name, channel, input.Recipients, false)
+		err := alert.createAlertNoti(ctx, userCred, input.Name, channel, input.Recipients, input.SilentPeriod, false)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("create notify[channel is %s]error", channel))
 		}
@@ -265,8 +270,8 @@ func (alert *SCommonAlert) customizeCreateNotis(ctx context.Context, userCred mc
 }
 
 func (alert *SCommonAlert) createAlertNoti(ctx context.Context, userCred mcclient.TokenCredential,
-	notiName, channel string, userIds []string, isSysNoti bool) error {
-	noti, err := NotificationManager.CreateOneCloudNotification(ctx, userCred, notiName, channel, userIds)
+	notiName, channel string, userIds []string, silentPeriod string, isSysNoti bool) error {
+	noti, err := NotificationManager.CreateOneCloudNotification(ctx, userCred, notiName, channel, userIds, silentPeriod)
 	if err != nil {
 		return errors.Wrap(err, "create notification")
 	}
@@ -746,6 +751,11 @@ func (alert *SCommonAlert) ValidateUpdateData(
 			data.Set("frequency", jsonutils.NewInt(freqSpec))
 		}
 	}
+	if silentPeriod, _ := data.GetString("silent_period"); len(silentPeriod) > 0 {
+		if _, err := time.ParseDuration(silentPeriod); err != nil {
+			return data, httperrors.NewInputParameterError("Invalid silent_period format: %s", silentPeriod)
+		}
+	}
 	//if recipients, _ := data.GetArray("recipients"); len(recipients) > 0 {
 	//	channelStr, _ := data.GetString("channel")
 	//	channel, _ := data.GetArray("channel")

pkg/monitor/models/nodealert.go

@@ -83,7 +83,7 @@ func (v1man *SV1AlertManager) CreateNotification(
 	channel string,
 	recipients string) (*SNotification, error) {
 	userIds := strings.Split(recipients, ",")
-	return NotificationManager.CreateOneCloudNotification(ctx, userCred, alertName, channel, userIds)
+	return NotificationManager.CreateOneCloudNotification(ctx, userCred, alertName, channel, userIds, "")
 }
 
 func (man *SNodeAlertManager) ValidateCreateData(

pkg/monitor/models/notification.go

@@ -165,7 +165,7 @@ func (man *SNotificationManager) CreateOneCloudNotification(
 	userCred mcclient.TokenCredential,
 	alertName string,
 	channel string,
-	userIds []string) (*SNotification, error) {
+	userIds []string, silentPeriod string) (*SNotification, error) {
 	settings := &monitor.NotificationSettingOneCloud{
 		Channel: channel,
 		UserIds: userIds,
@@ -179,6 +179,10 @@ func (man *SNotificationManager) CreateOneCloudNotification(
 		Type:     monitor.AlertNotificationTypeOneCloud,
 		Settings: jsonutils.Marshal(settings),
 	}
+	if silentPeriod != "" {
+		duration, _ := time.ParseDuration(silentPeriod)
+		input.Frequency = duration / time.Second
+	}
 	obj, err := db.DoCreate(man, ctx, userCred, nil, input.JSON(input), userCred)
 	if err != nil {
 		return nil, errors.Wrapf(err, "create notification input: %s", input.JSON(input))