Merge pull request #8986 from swordqiu/automated-cherry-pick-of-#8968-upstream-release-3.4

Automated cherry pick of #8968: fix(region,esxiagent): allow host behind a nat

.gitignore

@@ -31,6 +31,7 @@ _output
 /structargtest
 .idea
 .vscode
+intellij.yaml
 GPATH
 GRTAGS
 GTAGS

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@cn/DEFAULT

@@ -2,7 +2,7 @@
 策略名称: {{.name}}
 触发时间: {{.start_time}}
 报警级别: {{.level}}
-触发条件: {{.description}}
+触发条件: {{.description | unescaped}}
 资源数量：{{len .matches}}
 资源名称：{{.resource_name}}
 

build/monitor/root/opt/yunion/share/notify_templates/monitor/content@en/DEFAULT

@@ -2,7 +2,7 @@
 AlertName: {{.name}}
 Time: {{.start_time}}
 Level: {{.level}}
-TriggerCondition: {{html .description}}
+TriggerCondition: {{.description | unescaped}}
 ResourceCount: {{len .matches}}
 ResourceName: {{.resource_name}}
 

pkg/apis/monitor/alertrecord.go

@@ -34,7 +34,9 @@ type AlertRecordCreateInput struct {
 
 type AlertRecordRule struct {
 	Metric          string `json:"metric"`
+	Measurement     string `json:"measurement"`
 	MeasurementDesc string `json:"measurement_desc"`
+	Field           string `json:"field"`
 	FieldDesc       string `json:"field_desc"`
 	// 比较运算符, 比如: >, <, >=, <=
 	Comparator string `json:"comparator"`

pkg/cloudcommon/db/jointbase.go

@@ -34,7 +34,7 @@ import (
 type SJointResourceBase struct {
 	SResourceBase
 
-	RowId int64 `primary:"true" auto_increment:"true"`
+	RowId int64 `primary:"true" auto_increment:"true" list:"user"`
 }
 
 type SJointResourceBaseManager struct {

pkg/cloudcommon/notifyclient/notify.go

@@ -91,7 +91,9 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 		if err != nil {
 			return nil, err
 		}
-		tmp, err := template.New(key).Parse(string(cont))
+		tmp := template.New(key)
+		tmp.Funcs(template.FuncMap{"unescaped": unescaped})
+		tmp, err = tmp.Parse(string(cont))
 		if err != nil {
 			return nil, err
 		}
@@ -100,6 +102,10 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 	return templatesTable[key], nil
 }
 
+func unescaped(str string) template.HTML {
+	return template.HTML(str)
+}
+
 func getContent(ctx context.Context, topic string, contType string, channel npk.TNotifyChannel, data jsonutils.JSONObject) (string, error) {
 	if channel == npk.NotifyByWebhook {
 		return "", nil

pkg/compute/guestdrivers/qcloud.go

@@ -136,7 +136,7 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		if sysDisk.SizeMb > 1024*1024 {
 			return nil, fmt.Errorf("The %s system disk size must be less than 1024GB", sysDisk.Backend)
 		}
-	case api.STORAGE_LOCAL_PRO:
+	case api.STORAGE_LOCAL_PRO, api.STORAGE_CLOUD_HSSD: //https://cloud.tencent.com/document/product/362/2353
 		return nil, fmt.Errorf("storage %s can not be system disk", sysDisk.Backend)
 	}
 

pkg/compute/hostdrivers/huawei.go

@@ -59,7 +59,15 @@ func (self *SHuaweiHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb
 
 func (self *SHuaweiHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	if len(guests) >= 1 {
-		return nil, httperrors.NewBadRequestError("Disk must be dettached")
+		if disk.DiskType == api.DISK_TYPE_SYS {
+			for _, g := range guests {
+				if g.Status != api.VM_READY {
+					return nil, httperrors.NewBadRequestError("Server %s must in status ready", g.GetName())
+				}
+			}
+		} else {
+			return nil, httperrors.NewBadRequestError("Disk must be detached")
+		}
 	}
 	return data, nil
 }

pkg/compute/models/elasticcache_instances.go

@@ -1164,7 +1164,7 @@ func (self *SElasticcache) GetAdminAccount() (*SElasticcacheAccount, error) {
 		}
 	}
 
-	return nil, fmt.Errorf("no admin account found for elastic cache %s", self.Id)
+	return nil, httperrors.NewNotFoundError("no admin account found for elastic cache %s", self.Id)
 }
 
 func (self *SElasticcache) StartResetPasswordTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error {

pkg/compute/models/elasticips.go

@@ -509,6 +509,7 @@ func (manager *SElasticipManager) newFromCloudEip(ctx context.Context, userCred
 	eip.ManagerId = provider.Id
 	eip.CloudregionId = region.Id
 	eip.ChargeType = extEip.GetInternetChargeType()
+	eip.Bandwidth = extEip.GetBandwidth()
 	if networkId := extEip.GetINetworkId(); len(networkId) > 0 {
 		network, err := db.FetchByExternalIdAndManagerId(NetworkManager, networkId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
 			wire := WireManager.Query().SubQuery()

pkg/compute/models/guest_actions.go

@@ -654,13 +654,16 @@ func (self *SGuest) ValidateAttachDisk(ctx context.Context, disk *SDisk) error {
 		}
 	}
 
-	attached, err := disk.isAttached()
-	if err != nil {
-		return httperrors.NewInternalServerError("isAttached check failed %s", err)
-	}
-	if attached {
-		return httperrors.NewInputParameterError("Disk %s has been attached", disk.Name)
+	if disk.IsLocal() {
+		attached, err := disk.isAttached()
+		if err != nil {
+			return httperrors.NewInternalServerError("isAttached check failed %s", err)
+		}
+		if attached {
+			return httperrors.NewInputParameterError("Disk %s has been attached", disk.Name)
+		}
 	}
+
 	if len(disk.GetPathAtHost(self.GetHost())) == 0 {
 		return httperrors.NewInputParameterError("Disk %s not belong the guest's host", disk.Name)
 	}
@@ -2073,6 +2076,12 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T
 			}
 			return nil, httperrors.NewBadRequestError("%v", err)
 		}
+		if _, err := db.Update(&ngn[0], func() error {
+			ngn[0].EipId = gn.EipId
+			return nil
+		}); err != nil {
+			return nil, err
+		}
 
 		return ngn, nil
 	}()

pkg/compute/models/guests.go

@@ -3703,11 +3703,17 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To
 		}
 
 		if disk != nil {
-			db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
-			db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
-			err = disk.RealDelete(ctx, userCred)
+			cnt, err := disk.GetGuestDiskCount()
 			if err != nil {
-				return err
+				return errors.Wrap(err, "disk.GetGuestDiskCount")
+			}
+			if cnt == 0 {
+				db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
+				db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
+				err = disk.RealDelete(ctx, userCred)
+				if err != nil {
+					return errors.Wrap(err, "disk.RealDelete")
+				}
 			}
 		}
 	}

pkg/compute/models/networks.go

@@ -1719,7 +1719,9 @@ func (self *SNetwork) PostCreate(ctx context.Context, userCred mcclient.TokenCre
 		}
 	} else {
 		self.SetStatus(userCred, api.NETWORK_STATUS_AVAILABLE, "")
-		self.ClearSchedDescCache()
+		if err := self.ClearSchedDescCache(); err != nil {
+			log.Errorf("network post create clear schedcache error: %v", err)
+		}
 	}
 }
 
@@ -1750,7 +1752,6 @@ func (self *SNetwork) RealDelete(ctx context.Context, userCred mcclient.TokenCre
 	DeleteResourceJointSchedtags(self, ctx, userCred)
 	db.OpsLog.LogEvent(self, db.ACT_DELOCATE, self.GetShortDesc(ctx), userCred)
 	self.SetStatus(userCred, api.NETWORK_STATUS_DELETED, "real delete")
-	self.ClearSchedDescCache()
 	networkinterfaces, err := self.GetNetworkInterfaces()
 	if err != nil {
 		return errors.Wrap(err, "GetNetworkInterfaces")
@@ -1771,7 +1772,11 @@ func (self *SNetwork) RealDelete(ctx context.Context, userCred mcclient.TokenCre
 			return errors.Wrapf(err, "reservedIps.Release %s(%d)", reservedIps[i].IpAddr, reservedIps[i].Id)
 		}
 	}
-	return self.SSharableVirtualResourceBase.Delete(ctx, userCred)
+	if err := self.SSharableVirtualResourceBase.Delete(ctx, userCred); err != nil {
+		return err
+	}
+	self.ClearSchedDescCache()
+	return nil
 }
 
 func (self *SNetwork) StartDeleteNetworkTask(ctx context.Context, userCred mcclient.TokenCredential) error {
@@ -1807,12 +1812,7 @@ func (self *SNetwork) isManaged() bool {
 }
 
 func (self *SNetwork) isOneCloudVpcNetwork() bool {
-	vpc := self.GetVpc()
-	region := self.GetRegion()
-	if region.Provider == api.CLOUD_PROVIDER_ONECLOUD && vpc.Id != api.DEFAULT_VPC_ID {
-		return true
-	}
-	return false
+	return IsOneCloudVpcResource(self)
 }
 
 func parseIpToIntArray(ip string) ([]int, error) {

pkg/compute/models/storagecachedimages.go

@@ -273,12 +273,14 @@ func (self *SStoragecachedimage) Detach(ctx context.Context, userCred mcclient.T
 }
 
 func (self *SStoragecachedimage) ValidateDeleteCondition(ctx context.Context) error {
-	cnt, err := self.getReferenceCount()
-	if err != nil {
-		return httperrors.NewInternalServerError("getReferenceCount fail %s", err)
-	}
-	if cnt > 0 {
-		return httperrors.NewNotEmptyError("Image is in use")
+	if self.Status != api.CACHED_IMAGE_STATUS_CACHE_FAILED {
+		cnt, err := self.getReferenceCount()
+		if err != nil {
+			return httperrors.NewInternalServerError("getReferenceCount fail %s", err)
+		}
+		if cnt > 0 {
+			return httperrors.NewNotEmptyError("Image is in use")
+		}
 	}
 	return self.SJointResourceBase.ValidateDeleteCondition(ctx)
 }

pkg/compute/models/vpcresource.go

@@ -32,6 +32,11 @@ import (
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
+type IVpcResource interface {
+	GetVpc() *SVpc
+	GetRegion() *SCloudregion
+}
+
 type SVpcResourceBase struct {
 	VpcId string `width:"36" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"vpc_id"`
 }
@@ -307,3 +312,18 @@ func (self *SVpcResourceBase) GetChangeOwnerCandidateDomainIds() []string {
 	}
 	return nil
 }
+
+func IsOneCloudVpcResource(res IVpcResource) bool {
+	vpc := res.GetVpc()
+	if vpc == nil {
+		return false
+	}
+	region := res.GetRegion()
+	if region == nil {
+		return false
+	}
+	if region.Provider == api.CLOUD_PROVIDER_ONECLOUD && vpc.Id != api.DEFAULT_VPC_ID {
+		return true
+	}
+	return false
+}

pkg/compute/models/wires.go

@@ -857,6 +857,10 @@ func (manager *SWireManager) InitializeData() error {
 	return nil
 }
 
+func (wire *SWire) isOneCloudVpcWire() bool {
+	return IsOneCloudVpcResource(wire)
+}
+
 func (wire *SWire) getEnabledHosts() []SHost {
 	hosts := make([]SHost, 0)
 
@@ -866,9 +870,13 @@ func (wire *SWire) getEnabledHosts() []SHost {
 	q := hostQuery.Query()
 	q = q.Join(hostwireQuery, sqlchemy.AND(sqlchemy.Equals(hostQuery.Field("id"), hostwireQuery.Field("host_id")),
 		sqlchemy.IsFalse(hostwireQuery.Field("deleted"))))
-	q = q.Filter(sqlchemy.Equals(hostwireQuery.Field("wire_id"), wire.Id))
 	q = q.Filter(sqlchemy.IsTrue(hostQuery.Field("enabled")))
 	q = q.Filter(sqlchemy.Equals(hostQuery.Field("host_status"), api.HOST_ONLINE))
+	if wire.isOneCloudVpcWire() {
+		q = q.Filter(sqlchemy.NOT(sqlchemy.IsNullOrEmpty(hostQuery.Field("ovn_version"))))
+	} else {
+		q = q.Filter(sqlchemy.Equals(hostwireQuery.Field("wire_id"), wire.Id))
+	}
 
 	err := db.FetchModelObjects(HostManager, q, &hosts)
 	if err != nil {
@@ -883,10 +891,9 @@ func (wire *SWire) clearHostSchedDescCache() error {
 	hosts := wire.getEnabledHosts()
 	if hosts != nil {
 		for i := 0; i < len(hosts); i += 1 {
-			err := hosts[i].ClearSchedDescCache()
-			if err != nil {
-				log.Errorf("%s", err)
-				return err
+			host := hosts[i]
+			if err := host.ClearSchedDescCache(); err != nil {
+				return errors.Wrapf(err, "wire %s clear host %s sched cache", wire.GetName(), host.GetName())
 			}
 		}
 	}

pkg/compute/tasks/elasticcache_account_create_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAccountCreateTask) OnInit(ctx context.Context, obj db.IS
 
 	self.SetStage("OnElasticcacheAccountCreateComplete", nil)
 	if err := region.GetDriver().RequestCreateElasticcacheAccount(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_account_delete_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAccountDeleteTask) OnInit(ctx context.Context, obj db.IS
 
 	self.SetStage("OnElasticcacheAccountDeleteComplete", nil)
 	if err := region.GetDriver().RequestDeleteElasticcacheAccount(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	} else {
 		err = db.DeleteModel(ctx, self.GetUserCred(), ea)

pkg/compute/tasks/elasticcache_account_reset_password_task.go

@@ -59,7 +59,7 @@ func (self *ElasticcacheAccountResetPasswordTask) OnInit(ctx context.Context, ob
 
 	self.SetStage("OnElasticcacheAccountResetPasswordComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheAccountResetPassword(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	} else {
 		logclient.AddActionLogWithStartable(self, ea, logclient.ACT_RESET_PASSWORD, nil, self.UserCred, true)

pkg/compute/tasks/elasticcache_acl_create_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAclCreateTask) OnInit(ctx context.Context, obj db.IStand
 
 	self.SetStage("OnElasticcacheAclCreateComplete", nil)
 	if err := region.GetDriver().RequestCreateElasticcacheAcl(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_acl_delete_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAclDeleteTask) OnInit(ctx context.Context, obj db.IStand
 
 	self.SetStage("OnElasticcacheAclDeleteComplete", nil)
 	if err := region.GetDriver().RequestDeleteElasticcacheAcl(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	} else {
 		err = db.DeleteModel(ctx, self.GetUserCred(), ea)

pkg/compute/tasks/elasticcache_acl_update_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAclUpdateTask) OnInit(ctx context.Context, obj db.IStand
 
 	self.SetStage("OnElasticcacheAclUpdateComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheAclUpdate(ctx, self.GetUserCred(), ea, self); err != nil {
-		self.taskFail(ctx, ea, jsonutils.Marshal(err))
+		self.taskFail(ctx, ea, jsonutils.NewString(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_allocate_public_connection_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheAllocatePublicConnectionTask) OnInit(ctx context.Context
 
 	self.SetStage("OnElasticcacheAllocatePublicConnectionComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheAllocatePublicConnection(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheAllocatePublicConnectionCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheAllocatePublicConnectionCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_backup_create_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheBackupCreateTask) OnInit(ctx context.Context, obj db.ISt
 
 	self.SetStage("OnElasticcacheBackupCreateComplete", nil)
 	if err := region.GetDriver().RequestCreateElasticcacheBackup(ctx, self.GetUserCred(), eb, self); err != nil {
-		self.taskFail(ctx, eb, jsonutils.Marshal(err))
+		self.taskFail(ctx, eb, jsonutils.NewString(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_backup_restore_instance_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheBackupRestoreInstanceTask) OnInit(ctx context.Context, o
 
 	self.SetStage("OnElasticcacheBackupRestoreInstanceComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheBackupRestoreInstance(ctx, self.GetUserCred(), eb, self); err != nil {
-		self.OnElasticcacheBackupRestoreInstanceCompleteFailed(ctx, eb, jsonutils.Marshal(err))
+		self.OnElasticcacheBackupRestoreInstanceCompleteFailed(ctx, eb, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_change_spec_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheChangeSpecTask) OnInit(ctx context.Context, obj db.IStan
 
 	self.SetStage("OnElasticcacheChangeSpecComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheChangeSpec(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheChangeSpecCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheChangeSpecCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_create_task.go

@@ -61,7 +61,7 @@ func (self *ElasticcacheCreateTask) OnSyncSecurityGroupComplete(ctx context.Cont
 	region := elasticcache.GetRegion()
 	self.SetStage("OnElasticcacheCreateComplete", nil)
 	if err := region.GetDriver().RequestCreateElasticcache(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.taskFail(ctx, elasticcache, jsonutils.Marshal(err))
+		self.taskFail(ctx, elasticcache, jsonutils.Marshal(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_flush_instance.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheFlushInstanceTask) OnInit(ctx context.Context, obj db.IS
 
 	self.SetStage("OnElasticcacheFlushInstanceComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheFlushInstance(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheFlushInstanceCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheFlushInstanceCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_parameter_update_task.go

@@ -60,7 +60,7 @@ func (self *ElasticcacheParameterUpdateTask) OnInit(ctx context.Context, obj db.
 
 	self.SetStage("OnElasticcacheParameterUpdateComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheUpdateInstanceParameters(ctx, self.GetUserCred(), iec.(*models.SElasticcache), self); err != nil {
-		self.OnElasticcacheParameterUpdateCompleteFailed(ctx, ep, jsonutils.Marshal(err))
+		self.OnElasticcacheParameterUpdateCompleteFailed(ctx, ep, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_release_public_connection_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheReleasePublicConnectionTask) OnInit(ctx context.Context,
 
 	self.SetStage("OnElasticcacheReleasePublicConnectionComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheReleasePublicConnection(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheReleasePublicConnectionCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheReleasePublicConnectionCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_restart_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheRestartTask) OnInit(ctx context.Context, obj db.IStandal
 
 	self.SetStage("OnElasticcacheRestartComplete", nil)
 	if err := region.GetDriver().RequestRestartElasticcache(ctx, self.GetUserCred(), ec, self); err != nil {
-		self.taskFail(ctx, ec, jsonutils.Marshal(err))
+		self.taskFail(ctx, ec, jsonutils.NewString(err.Error()))
 		return
 	} else {
 		logclient.AddActionLogWithStartable(self, ec, logclient.ACT_VM_RESTART, nil, self.UserCred, true)

pkg/compute/tasks/elasticcache_set_maintain_time_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheSetMaintainTimeTask) OnInit(ctx context.Context, obj db.
 
 	self.SetStage("OnElasticcacheSetMaintainTimeComplete", nil)
 	if err := region.GetDriver().RequestElasticcacheSetMaintainTime(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheSetMaintainTimeCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheSetMaintainTimeCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/compute/tasks/elasticcache_sync_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheSyncTask) OnInit(ctx context.Context, obj db.IStandalone
 
 	self.SetStage("OnElasticcacheSyncComplete", nil)
 	if err := region.GetDriver().RequestSyncElasticcache(ctx, self.GetUserCred(), ec, self); err != nil {
-		self.taskFail(ctx, ec, jsonutils.Marshal(err))
+		self.taskFail(ctx, ec, jsonutils.NewString(err.Error()))
 		return
 	} else {
 		logclient.AddActionLogWithStartable(self, ec, logclient.ACT_SYNC_CONF, "", self.UserCred, true)

pkg/compute/tasks/elasticcache_syncstatus_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheSyncstatusTask) OnInit(ctx context.Context, obj db.IStan
 	self.SetStage("OnElasticcacheSyncStatusComplete", nil)
 	err := region.GetDriver().RequestSyncElasticcacheStatus(ctx, self.GetUserCred(), cache, self)
 	if err != nil {
-		self.taskFailed(ctx, cache, jsonutils.Marshal(err))
+		self.taskFailed(ctx, cache, jsonutils.NewString(err.Error()))
 		return
 	}
 }

pkg/compute/tasks/elasticcache_update_auth_mode_task.go

@@ -54,7 +54,7 @@ func (self *ElasticcacheUpdateAuthModeTask) OnInit(ctx context.Context, obj db.I
 
 	self.SetStage("OnElasticcacheUpdateAuthModeComplete", nil)
 	if err := region.GetDriver().RequestUpdateElasticcacheAuthMode(ctx, self.GetUserCred(), elasticcache, self); err != nil {
-		self.OnElasticcacheUpdateAuthModeCompleteFailed(ctx, elasticcache, jsonutils.Marshal(err))
+		self.OnElasticcacheUpdateAuthModeCompleteFailed(ctx, elasticcache, jsonutils.NewString(err.Error()))
 		return
 	}
 

pkg/hostman/hostinfo/hostinfo.go

@@ -1578,12 +1578,17 @@ func (h *SHostInfo) stop() {
 }
 
 func (h *SHostInfo) unregister() {
-	h.stopped = true
-	_, err := modules.Hosts.PerformAction(
-		h.GetSession(), h.HostId, "offline", nil)
-	if err != nil {
-		log.Errorln(err)
+	for {
+		_, err := modules.Hosts.PerformAction(
+			h.GetSession(), h.HostId, "offline", nil)
+		if err != nil {
+			log.Errorf("put host offline failed: %s", err)
+			time.Sleep(time.Second * 1)
+		} else {
+			break
+		}
 	}
+	h.stopped = true
 }
 
 func (h *SHostInfo) OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3) {

pkg/mcclient/options/servers.go

@@ -473,6 +473,8 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre
 		AutoPrepaidRecycle: opts.AutoPrepaidRecycle,
 		EipBw:              opts.EipBw,
 		EipChargeType:      opts.EipChargeType,
+		PublicIpBw:         opts.PublicIpBw,
+		PublicIpChargeType: opts.PublicIpChargeType,
 		Eip:                opts.Eip,
 		EnableCloudInit:    opts.EnableCloudInit,
 		OsType:             opts.OsType,

pkg/monitor/alerting/conditions/query.go

@@ -219,11 +219,25 @@ func (c *QueryCondition) NewEvalMatch(context *alerting.EvalContext, series tsdb
 	evalMatch.Tags = c.filterTags(series.Tags, *alertDetails)
 	evalMatch.Value = value
 	evalMatch.ValueStr = c.RationalizeValueFromUnit(*value, alertDetails.FieldDescription.Unit, alertDetails.FieldOpt)
-	evalMatch.MeasurementDesc = alertDetails.MeasurementDisplayName
-	evalMatch.FieldDesc = alertDetails.FieldDescription.DisplayName
+	c.newRuleDescription(context, alertDetails)
 	return evalMatch, nil
 }
 
+func (c *QueryCondition) newRuleDescription(context *alerting.EvalContext, alertDetails *monitor.CommonAlertMetricDetails) {
+	ruleDes := alerting.RuleDescription{
+		AlertRecordRule: monitor.AlertRecordRule{
+			Metric:          fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field),
+			Measurement:     alertDetails.Measurement,
+			MeasurementDesc: alertDetails.MeasurementDisplayName,
+			Field:           alertDetails.Field,
+			FieldDesc:       alertDetails.FieldDescription.DisplayName,
+			Comparator:      alertDetails.Comparator,
+			Threshold:       c.RationalizeValueFromUnit(alertDetails.Threshold, alertDetails.FieldDescription.Unit, ""),
+		},
+	}
+	context.RuleDescription = &ruleDes
+}
+
 var fileSize = []string{"bps", "Bps", "byte"}
 
 func (c *QueryCondition) RationalizeValueFromUnit(value float64, unit string, opt string) string {

pkg/monitor/alerting/conditions/suggestrulereducer.go

@@ -33,8 +33,8 @@ func NewSuggestRuleReducer(t string, duration time.Duration) Reducer {
 }
 
 func (s *suggestRuleReducer) Reduce(series *tsdb.TimeSeries) *float64 {
-	if int(s.duration.Seconds()) > len(series.Points) {
-		return nil
-	}
+	/*if int(s.duration.Seconds()) > len(series.Points) {
+		return nil, nil
+	}*/
 	return s.queryReducer.Reduce(series)
 }

pkg/monitor/alerting/eval_context.go

@@ -42,6 +42,7 @@ type EvalContext struct {
 	StartTime          time.Time
 	EndTime            time.Time
 	Rule               *Rule
+	RuleDescription    *RuleDescription
 
 	NoDataFound    bool
 	PrevAlertState monitor.AlertStateType
@@ -50,6 +51,10 @@ type EvalContext struct {
 	UserCred mcclient.TokenCredential
 }
 
+type RuleDescription struct {
+	monitor.AlertRecordRule
+}
+
 // NewEvalContext is the EvalContext constructor.
 func NewEvalContext(alertCtx context.Context, userCred mcclient.TokenCredential, rule *Rule) *EvalContext {
 	return &EvalContext{

pkg/monitor/alerting/notifier.go

@@ -17,7 +17,6 @@ package alerting
 import (
 	"database/sql"
 	"fmt"
-	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -143,32 +142,47 @@ func (n *notificationService) getNeededNotifiers(nIds []string, evalCtx *EvalCon
 		}
 	}
 	if shouldNotify {
-		var matches []*monitor.EvalMatch
-		if evalCtx.Firing {
-			matches = evalCtx.EvalMatches
-		} else {
-			matches = evalCtx.AlertOkEvalMatches
-		}
-		recordCreateInput := monitor.AlertRecordCreateInput{
-			StandaloneResourceCreateInput: apis.StandaloneResourceCreateInput{
-				GenerateName: evalCtx.Rule.Name,
-			},
-			AlertId:   evalCtx.Rule.Id,
-			Level:     evalCtx.Rule.Level,
-			State:     string(evalCtx.Rule.State),
-			EvalData:  matches,
-			AlertRule: newAlertRecordRule(evalCtx),
-		}
-		createData := recordCreateInput.JSON(recordCreateInput)
-		record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred)
-		if err != nil {
-			log.Errorf("create alert record err:%v", err)
-		}
-		record.PostCreate(evalCtx.Ctx, evalCtx.UserCred, evalCtx.UserCred, nil, createData)
+		n.createAlertRecordWhenNotify(evalCtx)
 	}
+	if !shouldNotify && evalCtx.shouldUpdateAlertState() && evalCtx.NoDataFound {
+		n.detachAlertResourceWhenNodata(evalCtx)
+	}
+
 	return result, nil
 }
 
+func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext) {
+	var matches []*monitor.EvalMatch
+	if evalCtx.Firing {
+		matches = evalCtx.EvalMatches
+	} else {
+		matches = evalCtx.AlertOkEvalMatches
+	}
+	recordCreateInput := monitor.AlertRecordCreateInput{
+		StandaloneResourceCreateInput: apis.StandaloneResourceCreateInput{
+			GenerateName: evalCtx.Rule.Name,
+		},
+		AlertId:   evalCtx.Rule.Id,
+		Level:     evalCtx.Rule.Level,
+		State:     string(evalCtx.Rule.State),
+		EvalData:  matches,
+		AlertRule: newAlertRecordRule(evalCtx),
+	}
+	createData := recordCreateInput.JSON(recordCreateInput)
+	record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred)
+	if err != nil {
+		log.Errorf("create alert record err:%v", err)
+	}
+	record.PostCreate(evalCtx.Ctx, evalCtx.UserCred, evalCtx.UserCred, nil, createData)
+}
+
+func (n *notificationService) detachAlertResourceWhenNodata(evalCtx *EvalContext) {
+	errs := models.CommonAlertManager.DetachAlertResourceByAlertId(evalCtx.Ctx, evalCtx.UserCred, evalCtx.Rule.Id)
+	if len(errs) != 0 {
+		log.Errorf("detachAlertResourceWhenNodata err:%#v", errors.NewAggregate(errs))
+	}
+}
+
 type NotifierPlugin struct {
 	Type               string
 	Factory            NotifierFactory
@@ -205,25 +219,14 @@ func InitNotifier(config NotificationConfig) (Notifier, error) {
 
 func newAlertRecordRule(evalCtx *EvalContext) monitor.AlertRecordRule {
 	alertRule := monitor.AlertRecordRule{}
+	if evalCtx.RuleDescription != nil {
+		alertRule = evalCtx.RuleDescription.AlertRecordRule
+	}
 	if evalCtx.Rule.Frequency < 60 {
 		alertRule.Period = fmt.Sprintf("%ds", evalCtx.Rule.Frequency)
 	} else {
 		alertRule.Period = fmt.Sprintf("%dm", evalCtx.Rule.Frequency/60)
 	}
-	ruleStr := evalCtx.Rule.Message
-	ruleElementArr := strings.Split(ruleStr, " ")
-	if len(ruleElementArr) == 3 {
-		alertRule.Metric = ruleElementArr[0]
-		alertRule.Comparator = ruleElementArr[1]
-		alertRule.Threshold = ruleElementArr[2]
-	}
-	if len(evalCtx.EvalMatches) != 0 {
-		alertRule.MeasurementDesc = evalCtx.EvalMatches[0].MeasurementDesc
-		alertRule.FieldDesc = evalCtx.EvalMatches[0].FieldDesc
-	}
-	if len(evalCtx.AlertOkEvalMatches) != 0 {
-		alertRule.MeasurementDesc = evalCtx.AlertOkEvalMatches[0].MeasurementDesc
-		alertRule.FieldDesc = evalCtx.AlertOkEvalMatches[0].FieldDesc
-	}
+
 	return alertRule
 }

pkg/monitor/models/commonalert.go

@@ -955,15 +955,20 @@ func (alert *SCommonAlert) StartDetachTask(ctx context.Context, userCred mcclien
 
 func (alert *SCommonAlert) DetachAlertResourceOnDisable(ctx context.Context,
 	userCred mcclient.TokenCredential) (errs []error) {
-	resources, err := GetAlertResourceManager().getResourceFromAlertId(alert.Id)
+	return CommonAlertManager.DetachAlertResourceByAlertId(ctx, userCred, alert.Id)
+}
+
+func (manager *SCommonAlertManager) DetachAlertResourceByAlertId(ctx context.Context,
+	userCred mcclient.TokenCredential, alertId string) (errs []error) {
+	resources, err := GetAlertResourceManager().getResourceFromAlertId(alertId)
 	if err != nil {
 		errs = append(errs, errors.Wrap(err, "getResourceFromAlert error"))
 		return
 	}
 	for _, resource := range resources {
-		err := resource.DetachAlert(ctx, userCred, alert.Id)
+		err := resource.DetachAlert(ctx, userCred, alertId)
 		if err != nil {
-			errs = append(errs, errors.Wrapf(err, "resource:%s DetachAlert:%s err", resource.Id, alert.Id))
+			errs = append(errs, errors.Wrapf(err, "resource:%s DetachAlert:%s err", resource.Id, alertId))
 		}
 	}
 	return

pkg/monitor/models/suggestsysrule.go

@@ -624,19 +624,19 @@ func showDuration(dur string) string {
 func showDuration_(dur int64, sign string) string {
 	var durUp, durSign int64
 	var upSign, durStr string
-	if sign == "s" && dur > 60 {
+	if sign == "s" && dur >= 60 {
 		upSign = "m"
 		durUp = dur / 60
 		durSign = dur % 60
 		durStr = showDuration_(durUp, upSign)
 	}
-	if sign == "m" && dur > 60 {
+	if sign == "m" && dur >= 60 {
 		upSign = "h"
 		durUp = dur / 60
 		durSign = dur % 60
 		durStr = showDuration_(durUp, upSign)
 	}
-	if sign == "h" && dur > 24 {
+	if sign == "h" && dur >= 24 {
 		upSign = "d"
 		durUp = dur / 24
 		durSign = dur % 24

pkg/monitor/service/service.go

@@ -45,7 +45,7 @@ import (
 
 func StartService() {
 	opts := &options.Options
-	common_options.ParseOptions(opts, os.Args, "alerter.conf", "alerter")
+	common_options.ParseOptions(opts, os.Args, "monitor.conf", "monitor")
 
 	commonOpts := &opts.CommonOptions
 	common_app.InitAuth(commonOpts, func() {

pkg/multicloud/esxi/manager.go

@@ -513,7 +513,11 @@ func (cli *SESXiClient) getPrivateId(idStr string) string {
 func (cli *SESXiClient) checkHostManagedByVCenter() error {
 	host, err := cli.FindHostByIp(cli.host)
 	if err != nil {
-		return err
+		if errors.Cause(err) == errors.ErrNotFound {
+			// host might be behind a NAT
+			return nil
+		}
+		return errors.Wrap(err, "cli.FindHostByIp")
 	}
 	if host.IsManagedByVCenter() {
 		return fmt.Errorf("ESXi host is managed by vcenter %s, please connect to vcenter instead for full management functions!", host.GetManagementServerIp())
@@ -541,18 +545,18 @@ func (cli *SESXiClient) FindHostByIp(hostIp string) (*SHost, error) {
 	hostRef, err := searchIndex.FindByIp(cli.context, nil, cli.getPrivateId(hostIp), false)
 	if err != nil {
 		log.Errorf("searchIndex.FindByIp fail %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "searchIndex.FindByIp")
 	}
 
 	if hostRef == nil {
-		return nil, fmt.Errorf("cannot find %s", cli.getPrivateId(hostIp))
+		return nil, errors.Wrapf(errors.ErrNotFound, "cannot find %s", cli.getPrivateId(hostIp))
 	}
 
 	var host mo.HostSystem
 	err = cli.reference2Object(hostRef.Reference(), HOST_SYSTEM_PROPS, &host)
 	if err != nil {
 		log.Errorf("reference2Object fail %s", err)
-		return nil, err
+		return nil, errors.Wrap(err, "cli.reference2Object")
 	}
 
 	h := NewHost(cli, &host, nil)

pkg/multicloud/qcloud/instance.go

@@ -863,6 +863,7 @@ func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) {
 		eip.AddressName = address
 		eip.AddressType = EIP_TYPE_WANIP
 		eip.AddressStatus = EIP_STATUS_BIND
+		eip.Bandwidth = self.InternetAccessible.InternetMaxBandwidthOut
 		return &eip, nil
 	}
 	return nil, nil

pkg/multicloud/qcloud/storage.go

@@ -154,5 +154,5 @@ func (self *SStorage) GetMountPoint() string {
 }
 
 func (self *SStorage) IsSysDiskStore() bool {
-	return true
+	return strings.ToLower(self.storageType) != api.STORAGE_CLOUD_HSSD
 }

pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go

@@ -156,6 +156,7 @@ func (p *DiskSchedtagPredicate) DoSelect(
 	res []ISchedtagCandidateResource,
 ) []ISchedtagCandidateResource {
 	return p.GetUsedStorages(res, input.(*diskW).Backend)
+
 }
 
 func (p *DiskSchedtagPredicate) GetCandidateResourceSortScore(selectRes ISchedtagCandidateResource) int64 {
@@ -195,8 +196,8 @@ func (p *DiskSchedtagPredicate) GetUsedStorages(res []ISchedtagCandidateResource
 	} else {
 		backendStorages = storages
 	}
-	if len(backendStorages) == 0 {
+	/*if len(backendStorages) == 0 {
 		backendStorages = storages
-	}
+	}*/
 	return backendStorages
 }

pkg/scheduler/manager/manager.go

@@ -97,6 +97,9 @@ func (sm *SchedulerManager) start() {
 }
 
 func (sm *SchedulerManager) schedule(info *api.SchedInfo) (*core.ScheduleResult, error) {
+	// force sync clean expire cache before do schedule
+	sm.ExpireManager.Trigger()
+
 	log.V(10).Infof("SchedulerManager do schedule, input: %#v", info)
 	task, err := sm.TaskManager.AddTask(sm, info)
 	if err != nil {

pkg/util/seclib2/seclib.go

@@ -101,7 +101,7 @@ func AnalyzePasswordStrenth(passwd string) PasswordStrength {
 			ps.Lowercases += 1
 		} else if strings.IndexByte(ALL_UPPERS, passwd[i]) >= 0 {
 			ps.Uppercases += 1
-		} else if strings.IndexByte(ALL_PUNC, passwd[i]) >= 0 {
+		} else if strings.IndexByte(PUNC, passwd[i]) >= 0 {
 			ps.Punctuats += 1
 		}
 	}

pkg/util/seclib2/seclib_test.go

@@ -31,7 +31,8 @@ func TestMeetComplxity(t *testing.T) {
 		want bool
 	}{
 		{"123456", false},
-		{"123abcABC!@#", true},
+		{"123abcABC!@#", false},
+		{"123abcABC-@=", true},
 	}
 	for _, c := range cases {
 		if c.want != MeetComplxity(c.in) {

pkg/vpcagent/models/modelset.go

@@ -15,6 +15,8 @@
 package models
 
 import (
+	"fmt"
+
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -33,7 +35,7 @@ type (
 	SecurityGroupRules map[string]*SecurityGroupRule
 	Elasticips         map[string]*Elasticip
 
-	Guestnetworks  map[string]*Guestnetwork  // key: guestId/ifname
+	Guestnetworks  map[string]*Guestnetwork  // key: rowId
 	Guestsecgroups map[string]*Guestsecgroup // key: guestId/secgroupId
 
 	DnsRecords map[string]*DnsRecord
@@ -332,7 +334,8 @@ func (set Guestnetworks) NewModel() db.IModel {
 
 func (set Guestnetworks) AddModel(i db.IModel) {
 	m := i.(*Guestnetwork)
-	set[m.GuestId+"/"+m.Ifname] = m
+	k := fmt.Sprintf("%d", m.RowId)
+	set[k] = m
 }
 
 func (set Guestnetworks) Copy() apihelper.IModelSet {